diff --git a/CHANGELOG.md b/CHANGELOG.md index 464913e9f..0106ec34e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Integrated Maestro validation into merlin. - The sample generation command now logs stdout, stderr, and cmd.sh to merlin_info. - Add the bank and walltime keywords to the batch slurm launch, these will not alter the lsf launch. @@ -17,7 +16,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Updated to new celery (4.4.5) syntax for signature return codes. - Corrected prior bugfix. -- Unfroze Maestro version dependency. - Slightly improved logic by using regex to match variable tokens. ## [1.6.1] diff --git a/merlin/celery.py b/merlin/celery.py index 463e25012..1bed40351 100644 --- a/merlin/celery.py +++ b/merlin/celery.py @@ -29,10 +29,7 @@ ############################################################################### """Updated celery configuration.""" -from __future__ import ( - absolute_import, - print_function, -) +from __future__ import absolute_import, print_function import logging import os @@ -43,10 +40,7 @@ from celery.signals import worker_process_init import merlin.common.security.encrypt_backend_traffic -from merlin.config import ( - broker, - results_backend, -) +from merlin.config import broker, results_backend from merlin.config.configfile import CONFIG from merlin.log_formatter import FORMATS from merlin.router import route_for_task diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py index 49dd77fd0..44963655e 100644 --- a/merlin/common/sample_index_factory.py +++ b/merlin/common/sample_index_factory.py @@ -33,10 +33,7 @@ """ from parse import parse -from merlin.common.sample_index import ( - MAX_SAMPLE, - SampleIndex, -) +from merlin.common.sample_index import MAX_SAMPLE, SampleIndex from merlin.utils import cd diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 695b6ce0d..4755730eb 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -29,25 +29,13 @@ ############################################################################### """Test tasks.""" -from __future__ import ( - absolute_import, - unicode_literals, -) +from __future__ import absolute_import, unicode_literals import logging import os -from celery import ( - chain, - chord, - group, - shared_task, - signature, -) -from celery.exceptions import ( - OperationalError, - TimeoutError, -) +from celery import chain, chord, group, shared_task, signature +from celery.exceptions import OperationalError, TimeoutError from merlin.common.abstracts.enums import ReturnCode from merlin.common.sample_index import uniform_directories diff --git a/merlin/config/broker.py b/merlin/config/broker.py index e8212ee60..629510fb8 100644 --- a/merlin/config/broker.py +++ b/merlin/config/broker.py @@ -36,10 +36,7 @@ import os from os.path import expanduser -from merlin.config.configfile import ( - CONFIG, - get_ssl_entries, -) +from merlin.config.configfile import CONFIG, get_ssl_entries try: diff --git a/merlin/config/results_backend.py b/merlin/config/results_backend.py index 4c5d6d3a4..a947a27dc 100644 --- a/merlin/config/results_backend.py +++ b/merlin/config/results_backend.py @@ -36,10 +36,7 @@ import logging import os -from merlin.config.configfile import ( - CONFIG, - get_ssl_entries, -) +from merlin.config.configfile import CONFIG, get_ssl_entries try: diff --git a/merlin/display.py b/merlin/display.py index b79607860..215d012bc 100644 --- a/merlin/display.py +++ b/merlin/display.py @@ -40,10 +40,7 @@ from tabulate import tabulate from merlin.ascii_art import banner_small -from merlin.config import ( - broker, - results_backend, -) +from merlin.config import broker, results_backend from merlin.config.configfile import default_config_info diff --git a/merlin/examples/workflows/feature_demo/scripts/learn.py b/merlin/examples/workflows/feature_demo/scripts/learn.py index 12570ada8..bb76fe1ea 100644 --- a/merlin/examples/workflows/feature_demo/scripts/learn.py +++ b/merlin/examples/workflows/feature_demo/scripts/learn.py @@ -2,7 +2,6 @@ import sys import numpy as np - from sklearn.ensemble import RandomForestRegressor diff --git a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py index d0d5eefc1..6c7641edb 100644 --- a/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/hpc_demo/cumulative_sample_processor.py @@ -1,10 +1,9 @@ import argparse import os import sys - -import matplotlib.pyplot as plt from concurrent.futures import ProcessPoolExecutor +import matplotlib.pyplot as plt import pandas as pd diff --git a/merlin/examples/workflows/hpc_demo/sample_collector.py b/merlin/examples/workflows/hpc_demo/sample_collector.py index 47c1862ed..622aadc65 100644 --- a/merlin/examples/workflows/hpc_demo/sample_collector.py +++ b/merlin/examples/workflows/hpc_demo/sample_collector.py @@ -1,7 +1,6 @@ import argparse import os import sys - from concurrent.futures import ProcessPoolExecutor diff --git a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py index d0d5eefc1..6c7641edb 100644 --- a/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py +++ b/merlin/examples/workflows/iterative_demo/cumulative_sample_processor.py @@ -1,10 +1,9 @@ import argparse import os import sys - -import matplotlib.pyplot as plt from concurrent.futures import ProcessPoolExecutor +import matplotlib.pyplot as plt import pandas as pd diff --git a/merlin/examples/workflows/iterative_demo/sample_collector.py b/merlin/examples/workflows/iterative_demo/sample_collector.py index 47c1862ed..622aadc65 100644 --- a/merlin/examples/workflows/iterative_demo/sample_collector.py +++ b/merlin/examples/workflows/iterative_demo/sample_collector.py @@ -1,7 +1,6 @@ import argparse import os import sys - from concurrent.futures import ProcessPoolExecutor diff --git a/merlin/examples/workflows/openfoam_wf/scripts/learn.py b/merlin/examples/workflows/openfoam_wf/scripts/learn.py index 3aecc4787..5c1f3ec16 100644 --- a/merlin/examples/workflows/openfoam_wf/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf/scripts/learn.py @@ -1,12 +1,12 @@ import argparse -import matplotlib.pyplot as plt import numpy as np - from joblib import dump from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error +import matplotlib.pyplot as plt + descript = """Using parameters to edit OpenFOAM parameters""" parser = argparse.ArgumentParser(description=descript) diff --git a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py index 3aecc4787..5c1f3ec16 100644 --- a/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py +++ b/merlin/examples/workflows/openfoam_wf_no_docker/scripts/learn.py @@ -1,12 +1,12 @@ import argparse -import matplotlib.pyplot as plt import numpy as np - from joblib import dump from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error +import matplotlib.pyplot as plt + descript = """Using parameters to edit OpenFOAM parameters""" parser = argparse.ArgumentParser(description=descript) diff --git a/merlin/main.py b/merlin/main.py index 2151eed46..8b779f898 100644 --- a/merlin/main.py +++ b/merlin/main.py @@ -44,20 +44,11 @@ ) from contextlib import suppress -from merlin import ( - VERSION, - router, -) +from merlin import VERSION, router from merlin.ascii_art import banner_small -from merlin.examples.generator import ( - list_examples, - setup_example, -) +from merlin.examples.generator import list_examples, setup_example from merlin.log_formatter import setup_logging -from merlin.spec.expansion import ( - RESERVED, - get_spec_with_expansion, -) +from merlin.spec.expansion import RESERVED, get_spec_with_expansion from merlin.spec.specification import MerlinSpec from merlin.study.study import MerlinStudy from merlin.utils import ARRAY_FILE_FORMATS diff --git a/merlin/spec/__init__.py b/merlin/spec/__init__.py index 8c48aaf99..397d1bd7d 100644 --- a/merlin/spec/__init__.py +++ b/merlin/spec/__init__.py @@ -27,8 +27,3 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ############################################################################### -import os - - -dirpath = os.path.dirname(os.path.abspath(__file__)) -SCHEMA_PATH = os.path.join(dirpath, "merlin_schemas.json") diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py index eadd21c65..1d60694d5 100644 --- a/merlin/spec/expansion.py +++ b/merlin/spec/expansion.py @@ -30,21 +30,12 @@ import logging from collections import ChainMap -from os.path import ( - expanduser, - expandvars, -) +from os.path import expanduser, expandvars from merlin.common.abstracts.enums import ReturnCode -from merlin.spec.override import ( - dump_with_overrides, - error_override_vars, -) +from merlin.spec.override import dump_with_overrides, error_override_vars from merlin.spec.specification import MerlinSpec -from merlin.utils import ( - contains_shell_ref, - contains_token, -) +from merlin.utils import contains_shell_ref, contains_token MAESTRO_RESERVED = {"SPECROOT", "WORKSPACE", "LAUNCHER"} diff --git a/merlin/spec/schemas/merlinsection.json b/merlin/spec/schemas/merlinsection.json deleted file mode 100644 index 6c81db89c..000000000 --- a/merlin/spec/schemas/merlinsection.json +++ /dev/null @@ -1,145 +0,0 @@ -{ - "MERLIN": { - "type": "object", - "properties": { - "resources": { - "type": "object", - "properties": { - "task_server": { - "type": "string", - "minLength": 1 - }, - "overlap": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "workers": { - "type": "object", - "additionalProperties": true, - "patternProperties": { - ".*": { - "type": "object", - "properties": { - "steps": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - }, - { - "type": "string" - } - ] - }, - "nodes": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - }, - { - "type": "null" - } - ] - }, - "batch": { - "anyOf": [ - { - "type": "object" - }, - { - "type": "null" - } - ] - }, - "args": { - "type": "string", - "minLength": 1 - }, - "machines": { - "type": "array", - "uniqueItems": true, - "items": { - "type": "string", - "minLength": 1 - } - } - }, - "additionalProperties": false, - "required": [ - "steps" - ] - } - } - } - }, - "additionalProperties": false - }, - "samples": { - "anyOf": [ - { - "type": "object", - "properties": { - "generate": { - "type": "object", - "properties": { - "cmd": { - "type": "string", - "minLength": 1 - } - }, - "additionalProperties": false - }, - "level_max_dirs": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "file": { - "type": "string", - "minLength": 1 - }, - "column_labels": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - } - } - }, - "additionalProperties": false - }, - { - "type": "null" - } - ] - } - }, - "additionalProperties": false, - "required": [ - "samples", - "resources" - ] - } -} diff --git a/merlin/spec/schemas/merlinspec.json b/merlin/spec/schemas/merlinspec.json deleted file mode 100644 index b88ff010c..000000000 --- a/merlin/spec/schemas/merlinspec.json +++ /dev/null @@ -1,397 +0,0 @@ -{ - "DESCRIPTION": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string", - "minLength": 1 - } - }, - "required": [ - "name", - "description" - ] - }, - "PARAM": { - "type": "object", - "properties": { - "values": { - "type": "array" - }, - "label": { - "type": "string", - "minLength": 1 - } - }, - "additionalProperties": false, - "required": [ - "values", - "label" - ] - }, - "BATCH": { - "type": "object", - "properties": { - "type": { - "type": "string", - "minLength": 1 - }, - "bank": { - "type": "string", - "minLength": 1 - }, - "queue": { - "type": "string", - "minLength": 1 - }, - "dry_run": { - "type": "boolean" - }, - "shell": { - "type": "string", - "minLength": 1 - }, - "flux_path": { - "type": "string", - "minLength": 1 - }, - "flux_start_opts": { - "type": "string", - "minLength": 1 - }, - "flux_exec_workers": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - } - }, - "launch_pre": { - "type": "string", - "minLength": 1 - }, - "launch_args": { - "type": "string", - "minLength": 1 - }, - "worker_launch": { - "type": "string", - "minLength": 1 - }, - "nodes": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "walltime": { - "type": "string", - "minLength": 1 - } - }, - "STUDY_STEP": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "description": { - "type": "string", - "minLength": 1 - }, - "run": { - "type": "object", - "properties": { - "cmd": { - "type": "string", - "minLength": 1 - }, - "depends": { - "type": "array", - "uniqueItems": true, - "items": { - "type": "string", - "minLength": 1 - } - }, - "pre": { - "type": "string", - "minLength": 1 - }, - "post": { - "type": "string", - "minLength": 1 - }, - "restart": { - "type": "string", - "minLength": 1 - }, - "nodes": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "procs": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "gpus": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "cores per task": { - "anyOf": [ - { - "type": "integer", - "minimum": 1 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "walltime": { - "type": "string", - "minLength": 1 - }, - "reservation": { - "type": "string", - "minLength": 1 - }, - "exclusive": { - "anyof": [ - { - "type": "boolean" - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "task_queue": { - "type": "string", - "minLength": 1 - }, - "shell": { - "type": "string", - "minLength": 1 - }, - "max_retries": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "gpus per task": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "batch": { - "type": "object", - "properties": { - "type": { - "type": "string", - "minLength": 1 - } - } - }, - "slurm": { - "type": "string", - "minLength": 1 - }, - "lsf": { - "type": "string", - "minLength": 1 - }, - "bind": { - "type": "string", - "minLength": 1 - }, - "num resource set": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "launch_distribution": { - "type": "string", - "minLength": 1 - }, - "exit_on_error": { - "anyOf": [ - { - "type": "integer", - "minimum": 0 - }, - { - "type": "string", - "pattern": "^\\$\\(\\w+\\)$" - } - ] - }, - "flux": { - "type": "string", - "minLength": 1 - } - }, - "additionalProperties": false, - "required": [ - "cmd" - ] - } - }, - "additionalProperties": false, - "required": [ - "name", - "description", - "run" - ] - }, - "ENV": { - "type": "object", - "properties": { - "variables": { - "type": "object" - }, - "labels": { - "type": "object" - }, - "sources": { - "type": "array" - }, - "dependencies": { - "type": "object", - "properties": { - "paths": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "path": { - "type": "string", - "minLength": 1 - } - }, - "required": [ - "name", - "path" - ], - "additionalProperties": false - } - }, - "git": { - "type": "array", - "items": { - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "path": { - "type": "string", - "minLength": 1 - }, - "url": { - "type": "string", - "minLength": 1 - }, - "tag": { - "type": "string", - "minLength": 1 - } - }, - "required": [ - "name", - "path", - "url" - ] - } - }, - "spack": { - "type": "object", - "properties": { - "name": { - "type": "string", - "minLength": 1 - }, - "package_name": { - "type": "string", - "minLength": 1 - } - }, - "required": [ - "name", - "package_name" - ] - } - } - } - }, - "additionalProperties": false - } -} diff --git a/merlin/spec/specification.py b/merlin/spec/specification.py index 9221d2af6..6c4b3f9e9 100644 --- a/merlin/spec/specification.py +++ b/merlin/spec/specification.py @@ -33,21 +33,14 @@ data from the Merlin specification file. To see examples of yaml specifications, run `merlin example`. """ -import json import logging import os -from contextlib import suppress from io import StringIO -import jsonschema import yaml -from maestrowf.specification.yamlspecification import YAMLSpecification +from maestrowf.datastructures import YAMLSpecification -from merlin.spec import ( - SCHEMA_PATH, - all_keys, - defaults, -) +from merlin.spec import all_keys, defaults def represent_none(self, _): @@ -91,7 +84,7 @@ def load_specification(cls, filepath, suppress_warning=True): spec.specroot = os.path.dirname(spec.path) spec.process_spec_defaults() if not suppress_warning: - spec.verify_merlin_block() + spec.warn_unrecognized_keys() return spec @classmethod @@ -154,6 +147,49 @@ def recurse(result, defaults): recurse(object_to_update, default_dict) + def warn_unrecognized_keys(self): + # check description + MerlinSpec.check_section("description", self.description, all_keys.DESCRIPTION) + + # check batch + MerlinSpec.check_section("batch", self.batch, all_keys.BATCH) + + # check env + MerlinSpec.check_section("env", self.environment, all_keys.ENV) + + # check parameters + for param, contents in self.globals.items(): + MerlinSpec.check_section("global.parameters", contents, all_keys.PARAMETER) + + # check steps + for step in self.study: + MerlinSpec.check_section(step["name"], step, all_keys.STUDY_STEP) + MerlinSpec.check_section( + step["name"] + ".run", step["run"], all_keys.STUDY_STEP_RUN + ) + + # check merlin + MerlinSpec.check_section("merlin", self.merlin, all_keys.MERLIN) + MerlinSpec.check_section( + "merlin.resources", self.merlin["resources"], all_keys.MERLIN_RESOURCES + ) + for worker, contents in self.merlin["resources"]["workers"].items(): + MerlinSpec.check_section( + "merlin.resources.workers " + worker, contents, all_keys.WORKER + ) + if self.merlin["samples"]: + MerlinSpec.check_section( + "merlin.samples", self.merlin["samples"], all_keys.SAMPLES + ) + + @staticmethod + def check_section(section_name, section, all_keys): + diff = set(section.keys()).difference(all_keys) + for extra in diff: + LOG.warn( + f"Unrecognized key '{extra}' found in spec section '{section_name}'." + ) + def dump(self): """ Dump this MerlinSpec to a yaml string. @@ -222,39 +258,3 @@ def get_worker_names(self): for worker in self.merlin["resources"]["workers"]: result.append(worker) return result - - def verify(self): - """Validate the specification.""" - - # load the MerlinSpec schema file - dirpath = os.path.dirname(os.path.abspath(__file__)) - schema_path = os.path.join(dirpath, "schemas") - schema_path = os.path.join(schema_path, "merlinspec.json") - with open(schema_path, "r") as json_file: - schemas = json.load(json_file) - - super().validate_schema("description", self.description, schemas["DESCRIPTION"]) - super().validate_schema("env", self.environment, schemas["ENV"]) - super().validate_schema("batch", self.batch, schemas["BATCH"]) - for step in self.study: - super().validate_schema( - f"study step {step['name']}", step, schemas["STUDY_STEP"] - ) - for param, contents in self.globals.items(): - super().validate_schema("global.params", contents, schemas["PARAM"]) - - LOG.debug("Spec verified. No errors found.") - - def verify_merlin_block(self): - """Validate the merlin block.""" - - # load the merlin block schema file - dirpath = os.path.dirname(os.path.abspath(__file__)) - schema_path = os.path.join(dirpath, "schemas") - schema_path = os.path.join(schema_path, "merlinsection.json") - with open(schema_path, "r") as json_file: - schemas = json.load(json_file) - - super().validate_schema("merlin", self.merlin, schemas["MERLIN"]) - - LOG.debug("Merlin block verified. No errors found.") diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py index dfd37a595..fd38ef16c 100644 --- a/merlin/study/celeryadapter.py +++ b/merlin/study/celeryadapter.py @@ -38,10 +38,7 @@ import time from contextlib import suppress -from merlin.study.batch import ( - batch_check_parallel, - batch_worker_launch, -) +from merlin.study.batch import batch_check_parallel, batch_worker_launch from merlin.utils import ( check_machines, get_procs, diff --git a/merlin/study/study.py b/merlin/study/study.py index ade49cb7f..0909501ed 100644 --- a/merlin/study/study.py +++ b/merlin/study/study.py @@ -41,21 +41,11 @@ from merlin.common.abstracts.enums import ReturnCode from merlin.spec import defaults -from merlin.spec.expansion import ( - determine_user_variables, - expand_line, -) -from merlin.spec.override import ( - dump_with_overrides, - error_override_vars, -) +from merlin.spec.expansion import determine_user_variables, expand_line +from merlin.spec.override import dump_with_overrides, error_override_vars from merlin.spec.specification import MerlinSpec from merlin.study.dag import DAG -from merlin.utils import ( - contains_token, - get_flux_cmd, - load_array_file, -) +from merlin.utils import contains_token, get_flux_cmd, load_array_file LOG = logging.getLogger(__name__) diff --git a/merlin/utils.py b/merlin/utils.py index 54d9cb526..31e11aa98 100644 --- a/merlin/utils.py +++ b/merlin/utils.py @@ -37,10 +37,7 @@ import re import socket import subprocess -from contextlib import ( - contextmanager, - suppress, -) +from contextlib import contextmanager, suppress from copy import deepcopy from types import SimpleNamespace diff --git a/requirements/release.txt b/requirements/release.txt index 787575ce9..6e1d330a3 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -3,7 +3,7 @@ celery[redis,sqlalchemy]>=4.4.5 coloredlogs cryptography importlib_resources; python_version < '3.7' -maestrowf>=1.1.8 +maestrowf==1.1.7dev0 numpy parse psutil>=5.1.0 diff --git a/setup.py b/setup.py index bd5894a61..7c7a0bac6 100644 --- a/setup.py +++ b/setup.py @@ -30,10 +30,7 @@ import os -from setuptools import ( - find_packages, - setup, -) +from setuptools import find_packages, setup version = __import__("merlin").VERSION diff --git a/tests/common/test_sample_index.py b/tests/common/test_sample_index.py index e5d34f32b..ba18db1e8 100644 --- a/tests/common/test_sample_index.py +++ b/tests/common/test_sample_index.py @@ -3,10 +3,7 @@ import unittest from contextlib import suppress -from merlin.common.sample_index_factory import ( - create_hierarchy, - read_hierarchy, -) +from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy TEST_DIR = "UNIT_TEST_SPACE" diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index ccd4cbef5..bd12017dc 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -40,10 +40,7 @@ from contextlib import suppress from glob import glob from re import search -from subprocess import ( - PIPE, - Popen, -) +from subprocess import PIPE, Popen from merlin.utils import get_flux_cmd @@ -389,7 +386,6 @@ def define_tests(): restart = "merlin restart" purge = "merlin purge" examples = "merlin/examples/workflows" - dev_examples = "merlin/examples/dev_workflows" demo = f"{examples}/feature_demo/feature_demo.yaml" simple = f"{examples}/simple_chain/simple_chain.yaml" slurm = f"{examples}/slurm/slurm_test.yaml" @@ -409,69 +405,6 @@ def define_tests(): ReturnCodeCond(), "local", ), - "local format 0": ( - f"merlin -lvl debug run {dev_examples}/full_format.yaml --local", - [ - RegexCond("Spec verified. No errors found."), - RegexCond("Merlin block verified. No errors found."), - ], - "local", - ), - "local format 1": ( - f"{run} {dev_examples}/bad_format1.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 2": ( - f"{run} {dev_examples}/bad_format2.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 3": ( - f"{run} {dev_examples}/bad_format3.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 4": ( - f"{run} {dev_examples}/bad_format4.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 5": ( - f"{run} {dev_examples}/bad_format5.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 6": ( - f"{run} {dev_examples}/bad_format6.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 7": ( - f"{run} {dev_examples}/bad_format7.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 8": ( - f"{run} {dev_examples}/bad_format8.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 9": ( - f"{run} {dev_examples}/bad_format9.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 10": ( - f"{run} {dev_examples}/bad_format10.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), - "local format 11": ( - f"{run} {dev_examples}/bad_format11.yaml --local", - ReturnCodeCond(expected_code=1), - "local", - ), "run-workers echo simple_chain": ( f"{workers} {simple} --echo", [ReturnCodeCond(), RegexCond(celery_regex)],