From 772ce867b363ac620d7316c2d866c045424da353 Mon Sep 17 00:00:00 2001 From: Benjamin Bay <48391872+ben-bay@users.noreply.github.com> Date: Fri, 18 Jun 2021 11:11:05 -0600 Subject: [PATCH 1/5] Delete .travis.yml --- .travis.yml | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 091eacd68..000000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -language: python -python: -- '3.6' -- '3.7' -- '3.8' -install: -- pip install --upgrade pip -- pip install -r requirements.txt -- pip install -r requirements/dev.txt -- pip install -r merlin/examples/workflows/feature_demo/requirements.txt -- pip install -e . -- pip install --upgrade sphinx -- merlin config -script: -- python -m pytest tests/unit/ -- python tests/integration/run_tests.py --verbose --local -- if [ "$TRAVIS_BRANCH" = "main" ]; then ./tests/integration/version_test.sh; fi From 9d946c98253b5ac5769d3c96b01244e942ac7153 Mon Sep 17 00:00:00 2001 From: Benjamin Bay Date: Fri, 18 Jun 2021 11:25:43 -0600 Subject: [PATCH 2/5] fixing merge mistake --- tests/integration/run_tests.py | 516 --------------------------------- 1 file changed, 516 deletions(-) diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py index 1d5035396..649e1ff3d 100644 --- a/tests/integration/run_tests.py +++ b/tests/integration/run_tests.py @@ -175,522 +175,6 @@ def run_tests(args, tests): return 1 -class Condition: - def ingest_info(self, info): - """ - This function allows child classes of Condition - to take in data AFTER a test is run. - """ - for key, val in info.items(): - setattr(self, key, val) - - @property - def passes(self): - print("Extend this class!") - return False - - -class ReturnCodeCond(Condition): - """ - A condition that some process must return 0 - as its return code. - """ - - def __init__(self, expected_code=0): - """ - :param `expected_code`: the expected return code - """ - self.expected_code = expected_code - - @property - def passes(self): - return self.return_code == self.expected_code - - -class NoStderrCond(Condition): - """ - A condition that some process have an empty - stderr string. - """ - - @property - def passes(self): - return self.stderr == "" - - -class RegexCond(Condition): - """ - A condition that some body of text MUST match a - given regular expression. Defaults to stdout. - """ - - def __init__(self, regex, negate=False): - """ - :param `regex`: a string regex pattern - """ - self.regex = regex - self.negate = negate - - def is_within(self, text): - """ - :param `text`: text in which to search for a regex match - """ - return search(self.regex, text) is not None - - @property - def passes(self): - if self.negate: - return not self.is_within(self.stdout) - return self.is_within(self.stdout) or self.is_within(self.stderr) - - -class StudyCond(Condition): - """ - An abstract condition that is aware of a study's name and output path. - """ - - def __init__(self, study_name, output_path): - """ - :param `study_name`: the name of a study - :param `output_path`: the $(OUTPUT_PATH) of a study - """ - self.study_name = study_name - self.output_path = output_path - self.dirpath_glob = f"{self.output_path}/{self.study_name}" f"_[0-9]*-[0-9]*" - - def glob(self, glob_string): - candidates = glob(glob_string) - if isinstance(candidates, list): - return sorted(candidates)[-1] - return candidates - - -class StepFileExistsCond(StudyCond): - """ - A StudyCond that checks for a particular file's existence. - """ - - def __init__(self, step, filename, study_name, output_path, params=False): - """ - :param `step`: the name of a step - :param `filename`: name of file to search for in step's workspace directory - :param `study_name`: the name of a study - :param `output_path`: the $(OUTPUT_PATH) of a study - """ - super().__init__(study_name, output_path) - self.step = step - self.filename = filename - self.params = params - - def file_exists(self): - param_glob = "" - if self.params: - param_glob = "*/" - glob_string = f"{self.dirpath_glob}/{self.step}/{param_glob}{self.filename}" - try: - filename = self.glob(glob_string) - except IndexError: - return False - return os.path.isfile(filename) - - @property - def passes(self): - return self.file_exists() - - -class StepFileContainsCond(StudyCond): - """ - A StudyCond that checks that a particular file contains a regex. - """ - - def __init__(self, step, filename, study_name, output_path, regex): - """ - :param `step`: the name of a step - :param `filename`: name of file to search for in step's workspace directory - :param `study_name`: the name of a study - :param `output_path`: the $(OUTPUT_PATH) of a study - """ - super().__init__(study_name, output_path) - self.step = step - self.filename = filename - self.regex = regex - - def contains(self): - glob_string = f"{self.dirpath_glob}/{self.step}/{self.filename}" - try: - filename = self.glob(glob_string) - with open(filename, "r") as textfile: - filetext = textfile.read() - return self.is_within(filetext) - except Exception: - return False - - def is_within(self, text): - """ - :param `text`: text in which to search for a regex match - """ - return search(self.regex, text) is not None - - @property - def passes(self): - return self.contains() - - -class ProvenanceCond(RegexCond): - """ - A condition that a Merlin provenance yaml spec - MUST contain a given regular expression. - """ - - def __init__(self, regex, name, output_path, provenance_type, negate=False): - """ - :param `regex`: a string regex pattern - :param `name`: the name of a study - :param `output_path`: the $(OUTPUT_PATH) of a study - """ - super().__init__(regex, negate=negate) - self.name = name - self.output_path = output_path - if provenance_type not in ["orig", "partial", "expanded"]: - raise ValueError( - f"Bad provenance_type '{provenance_type}' in ProvenanceCond!" - ) - self.prov_type = provenance_type - - def is_within(self): - """ - Uses glob to find the correct provenance yaml spec. - Returns True if that file contains a match to this - object's self.regex string. - """ - filepath = ( - f"{self.output_path}/{self.name}" - f"_[0-9]*-[0-9]*/merlin_info/{self.name}.{self.prov_type}.yaml" - ) - filename = sorted(glob(filepath))[-1] - with open(filename, "r") as _file: - text = _file.read() - return super().is_within(text) - - @property - def passes(self): - if self.negate: - return not self.is_within() - return self.is_within() - - -def define_tests(): - """ - Returns a dictionary of tests, where the key - is the test's name, and the value is a tuple - of (shell command, condition(s) to satisfy). - """ - celery_regex = r"(srun\s+.*)?celery\s+(-A|--app)\s+merlin\s+worker\s+.*" - - # shortcut string variables - workers = "merlin run-workers" - run = "merlin run" - restart = "merlin restart" - purge = "merlin purge" - examples = "merlin/examples/workflows" - dev_examples = "merlin/examples/dev_workflows" - demo = f"{examples}/feature_demo/feature_demo.yaml" - demo_pgen = f"{examples}/feature_demo/scripts/pgen.py" - simple = f"{examples}/simple_chain/simple_chain.yaml" - slurm = f"{examples}/slurm/slurm_test.yaml" - slurm_restart = f"{examples}/slurm/slurm_par_restart.yaml" - flux = f"{examples}/flux/flux_test.yaml" - flux_restart = f"{examples}/flux/flux_par_restart.yaml" - lsf = f"{examples}/lsf/lsf_par.yaml" - black = "black --check --target-version py36" - config_dir = "./CLI_TEST_MERLIN_CONFIG" - - return { - "merlin": ("merlin", ReturnCodeCond(1), "local"), - "merlin help": ("merlin --help", ReturnCodeCond(), "local"), - "merlin version": ("merlin --version", ReturnCodeCond(), "local"), - "merlin config": ( - f"merlin config -o {config_dir}; rm -rf {config_dir}", - ReturnCodeCond(), - "local", - ), - "local minimum_format": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../{dev_examples}/minimum_format.yaml --local", - StepFileExistsCond( - "step1", - "MERLIN_FINISHED", - "minimum_format", - OUTPUT_DIR, - params=False, - ), - "local", - ), - "local no_description": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_description.yaml --local", - ReturnCodeCond(1), - "local", - ), - "local no_steps": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_steps.yaml --local", - ReturnCodeCond(1), - "local", - ), - "local no_study": ( - f"mkdir {OUTPUT_DIR} ; cd {OUTPUT_DIR} ; merlin run ../merlin/examples/dev_workflows/no_study.yaml --local", - ReturnCodeCond(1), - "local", - ), - "run-workers echo simple_chain": ( - f"{workers} {simple} --echo", - [ReturnCodeCond(), RegexCond(celery_regex)], - "local", - ), - "run-workers echo feature_demo": ( - f"{workers} {demo} --echo", - [ReturnCodeCond(), RegexCond(celery_regex)], - "local", - ), - "run-workers echo slurm_test": ( - f"{workers} {slurm} --echo", - [ReturnCodeCond(), RegexCond(celery_regex)], - "local", - ), - "run-workers echo flux_test": ( - f"{workers} {flux} --echo", - [ReturnCodeCond(), RegexCond(celery_regex)], - "local", - ), - "run-workers echo override feature_demo": ( - f"{workers} {demo} --echo --vars VERIFY_QUEUE=custom_verify_queue", - [ReturnCodeCond(), RegexCond("custom_verify_queue")], - "local", - ), - "run feature_demo": (f"{run} {demo}", ReturnCodeCond()), - "purge feature_demo": (f"{purge} {demo} -f", ReturnCodeCond()), - "dry feature_demo": ( - f"{run} {demo} --local --dry --vars OUTPUT_PATH=./{OUTPUT_DIR}", - [ - StepFileExistsCond( - "verify", - "verify_*.sh", - "feature_demo", - OUTPUT_DIR, - params=True, - ), - ReturnCodeCond(), - ], - "local", - ), - "restart local simple_chain": ( - f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; {restart} $(find ./{OUTPUT_DIR} -type d -name 'simple_chain_*') --local", - ReturnCodeCond(), - "local", - ), - "local simple_chain": ( - f"{run} {simple} --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - ReturnCodeCond(), - "local", - ), - "local restart": ( - f"{run} {dev_examples}/restart.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileExistsCond( - "final_check_for_no_hard_fails", - "MERLIN_FINISHED", - "restart", - OUTPUT_DIR, - params=False, - ), - "local", - ), - "local restart_shell": ( - f"{run} {dev_examples}/restart_shell.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileExistsCond( - "step2", - "MERLIN_FINISHED", - "restart_shell", - OUTPUT_DIR, - params=False, - ), - "local", - ), - "example failure": (f"merlin example failure", RegexCond("not found"), "local"), - "example simple_chain": ( - f"merlin example simple_chain ; {run} simple_chain.yaml --local --vars OUTPUT_PATH=./{OUTPUT_DIR} ; rm simple_chain.yaml", - ReturnCodeCond(), - "local", - ), - "dry launch slurm": ( - f"{run} {slurm} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileContainsCond( - "runs", "*/runs.slurm.sh", "slurm_test", OUTPUT_DIR, "srun " - ), - "local", - ), - "dry launch flux": ( - f"{run} {flux} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileContainsCond( - "runs", - "*/runs.slurm.sh", - "flux_test", - OUTPUT_DIR, - get_flux_cmd("flux", no_errors=True), - ), - "local", - ), - "dry launch lsf": ( - f"{run} {lsf} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileContainsCond( - "runs", "*/runs.slurm.sh", "lsf_par", OUTPUT_DIR, "jsrun " - ), - "local", - ), - "dry launch slurm restart": ( - f"{run} {slurm_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileContainsCond( - "runs", - "*/runs.restart.slurm.sh", - "slurm_par_restart", - OUTPUT_DIR, - "srun ", - ), - "local", - ), - "dry launch flux restart": ( - f"{run} {flux_restart} --dry --local --no-errors --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR}", - StepFileContainsCond( - "runs_rs", - "*/runs_rs.restart.slurm.sh", - "flux_par_restart", - OUTPUT_DIR, - get_flux_cmd("flux", no_errors=True), - ), - "local", - ), - "local override feature_demo": ( - f"{run} {demo} --vars N_SAMPLES=2 OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ - ReturnCodeCond(), - ProvenanceCond( - regex="HELLO: \$\(SCRIPTS\)/hello_world.py", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="orig", - ), - ProvenanceCond( - regex="name: \$\(NAME\)", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="partial", - ), - ProvenanceCond( - regex="studies/feature_demo_", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="partial", - ), - ProvenanceCond( - regex="name: feature_demo", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - ), - ProvenanceCond( - regex="\$\(NAME\)", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - negate=True, - ), - # StepFileExistsCond( - # "verify", - # "MERLIN_FINISHED", - # "feature_demo", - # OUTPUT_DIR, - # params=True, - # ), - ], - "local", - ), - # "local restart expand name": ( - # f"{run} {demo} --local --vars OUTPUT_PATH=./{OUTPUT_DIR} NAME=test_demo ; {restart} $(find ./{OUTPUT_DIR} -type d -name 'test_demo_*') --local", - # [ - # ReturnCodeCond(), - # ProvenanceCond( - # regex="name: test_demo", - # name="test_demo", - # output_path=OUTPUT_DIR, - # provenance_type="expanded", - # ), - # StepFileExistsCond( - # "merlin_info", "test_demo.expanded.yaml", "test_demo", OUTPUT_DIR, params=True, - # ), - # ], - # "local", - # ), - "local csv feature_demo": ( - f"echo 42.0,47.0 > foo_testing_temp.csv; {run} {demo} --samplesfile foo_testing_temp.csv --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; rm -f foo_testing_temp.csv", - [RegexCond("1 sample loaded."), ReturnCodeCond()], - "local", - ), - "local tab feature_demo": ( - f"echo '42.0\t47.0\n7.0 5.3' > foo_testing_temp.tab; {run} {demo} --samplesfile foo_testing_temp.tab --vars OUTPUT_PATH=./{OUTPUT_DIR} --local; rm -f foo_testing_temp.tab", - [RegexCond("2 samples loaded."), ReturnCodeCond()], - "local", - ), - "local pgen feature_demo": ( - f"{run} {demo} --pgen {demo_pgen} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local", - [ - ProvenanceCond( - regex="\[0.3333333", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - ), - ProvenanceCond( - regex="\[0.5", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - negate=True, - ), - ReturnCodeCond(), - ], - "local", - ), - # "local provenance spec equality": ( - # f"{run} {simple} --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cp $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml') ./{OUTPUT_DIR}/FILE1 ; rm -rf ./{OUTPUT_DIR}/simple_chain_* ; {run} ./{OUTPUT_DIR}/FILE1 --vars OUTPUT_PATH=./{OUTPUT_DIR} --local ; cmp ./{OUTPUT_DIR}/FILE1 $(find ./{OUTPUT_DIR}/simple_chain_*/merlin_info -type f -name 'simple_chain.expanded.yaml')", - # ReturnCodeCond(), - # "local", - # ), - # "black check merlin": (f"{black} merlin/", ReturnCodeCond(), "local"), - # "black check tests": (f"{black} tests/", ReturnCodeCond(), "local"), - "deplic no GNU": ( - f"deplic ./", - [RegexCond("GNU", negate=True), RegexCond("GPL", negate=True)], - "local", - ), - "distributed feature_demo": ( - f"{run} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers ; {workers} {demo} --vars OUTPUT_PATH=./{OUTPUT_DIR} WORKER_NAME=cli_test_demo_workers", - [ - ReturnCodeCond(), - ProvenanceCond( - regex="cli_test_demo_workers:", - name="feature_demo", - output_path=OUTPUT_DIR, - provenance_type="expanded", - ), - StepFileExistsCond( - "verify", - "MERLIN_FINISHED", - "feature_demo", - OUTPUT_DIR, - params=True, - ), - ], - ), - } - - def setup_argparse(): parser = argparse.ArgumentParser(description="run_tests cli parser") parser.add_argument( From 9cf1e734a6984c6fcec81329224e050a5a141e62 Mon Sep 17 00:00:00 2001 From: Benjamin Bay Date: Fri, 18 Jun 2021 11:32:35 -0600 Subject: [PATCH 3/5] updated actions command --- .github/workflows/push-pr_workflow.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 689ab6b5a..72114a145 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -19,8 +19,8 @@ jobs: - name: Install dependencies run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip3 install --upgrade pip + if [ -f requirements.txt ]; then pip3 install -r requirements.txt; fi pip3 install -r requirements/dev.txt - name: Install merlin to run unit tests @@ -42,8 +42,8 @@ jobs: - name: Run pytest over unit test suite run: | - python -m pytest tests/ + python -m pytest tests/unit/ - name: Run integration test suite, locally run: | - python tests/integration/run_tests.py --verbose --local \ No newline at end of file + python tests/integration/run_tests.py --verbose --local From bbd78283062cb510e681b0bb8c5f3e1416ce586a Mon Sep 17 00:00:00 2001 From: Benjamin Bay Date: Fri, 18 Jun 2021 11:33:00 -0600 Subject: [PATCH 4/5] updated actions command --- .github/workflows/push-pr_workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 72114a145..096f4430e 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -19,7 +19,7 @@ jobs: - name: Install dependencies run: | - python -m pip3 install --upgrade pip + python3 -m pip3 install --upgrade pip if [ -f requirements.txt ]; then pip3 install -r requirements.txt; fi pip3 install -r requirements/dev.txt @@ -42,8 +42,8 @@ jobs: - name: Run pytest over unit test suite run: | - python -m pytest tests/unit/ + python3 -m pytest tests/unit/ - name: Run integration test suite, locally run: | - python tests/integration/run_tests.py --verbose --local + python3 tests/integration/run_tests.py --verbose --local From 7ee964f6eed62d97d5cd67b323c68e1a89837b6b Mon Sep 17 00:00:00 2001 From: Benjamin Bay Date: Fri, 18 Jun 2021 11:34:34 -0600 Subject: [PATCH 5/5] updated actions command --- .github/workflows/push-pr_workflow.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-pr_workflow.yml b/.github/workflows/push-pr_workflow.yml index 096f4430e..7edec65d9 100644 --- a/.github/workflows/push-pr_workflow.yml +++ b/.github/workflows/push-pr_workflow.yml @@ -19,8 +19,8 @@ jobs: - name: Install dependencies run: | - python3 -m pip3 install --upgrade pip - if [ -f requirements.txt ]; then pip3 install -r requirements.txt; fi + python3 -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi pip3 install -r requirements/dev.txt - name: Install merlin to run unit tests