diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md
new file mode 100644
index 000000000..20faf2602
--- /dev/null
+++ b/.github/CODE_OF_CONDUCT.md
@@ -0,0 +1,47 @@
+# Merlin Community Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the Merlin project or its community. Examples of representing the project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of the project may be further defined and clarified by Merlin maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at merlin@llnl.gov. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/
+
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
new file mode 100644
index 000000000..e2732be85
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -0,0 +1,35 @@
+---
+name: Bug report
+about: Create a report to help Merlin improve
+title: "[BUG]"
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Please answer these questions to help us pinpoint the problem**
+- Does the problem occur in `merlin run --local` mode, distributed mode or neither?
+- If a distributed problem, which backend and queue servers are you using? How are they configured?
+- On what machines/architectures are you running merlin? Is this bug on a specific machine or can you reproduce it elsewhere?
+
+**Please run `merlin info` and paste the results here:**
+
+
+**Additional context**
+Add any other context about the problem here.
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 000000000..df1d1b1c0
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for Merlin
+title: "[FEATURE]"
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 000000000..5d1fb6d56
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,24 @@
+language: python
+python:
+ - "3.6"
+ - "3.7"
+ - "3.8"
+install:
+ - "pip install --upgrade pip"
+ - "pip install cryptography"
+ - "pip install -r requirements.txt"
+ - "pip install -e ."
+ - "pip install -r requirements/mysql.txt"
+ - "pip install --upgrade sphinx"
+script:
+ - "merlin config"
+ - "python -m pytest tests/"
+ - "python tests/integration/run_tests.py --verbose --ids 1 2 3 4 5 6 7 8 9 12 13 18 19 20"
+deploy:
+ provider: pypi
+ user: "__token__"
+ password:
+ secure: Bd4xw0sEPixUiOwnl+5MYQaSX8SHKc0Dughg2Ls/B/e3NcvmCbYeBEz+2yclByffdNkxrQCTzMI90gZ5i7aKBuzEZ4E/RHXytwoLWOIKw3rE5jun44Vb9anLz2V+hEF6DSZ47fe+5re0D/TKCjtP7KTqLTDDdgg6+f648IbiE2CCmJXXeia48MzkUlLHYRbIuyWXGhaQasZwIoZQ872+xX0n8/3p+bV6BZ415DwJSsEdFmDu/F9drSVLUV6monU4eIfVoveos4HPH76J5WDMlwq6ZMaxF5E88ExiM9JJ4KNrrxBgLFS2I0BZmI92edHqtSs6TWz7Qo4wvW/6emosQjs6EWS8JmhSMdMFyhBos+Mx6HJYqJb4Nxcx9vclcpWSSzKSKUkAKX7bN5+v2Oy5BXfCdLBxWY+et+6ZY7/SzutOfgPmEz7BG7YiC9vZAGYpas4H3C6sOAdIb8y2lvL9Vht1+94GITZ0nFzdmpNGmAN6g9BAPvrEVWnZKdy0qpcBIpS/KW0N5/8jG74D0ouEDRgH5ahV5+RoyzzGrShC2XBHne4bGHb59DHWBRJBo2bS2Df0+gZPsd15kfjOCyZNHUIni2yQ83GgGc7hznLJd/UIvvPTGG42LjppzER5kimGor8BZ3B58dKrHqqFgRyAowOgoVlhd01rMi4a1QwS6ao=
+ on:
+ tags: true
+ branch: master
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 000000000..b2989d501
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,15 @@
+# Contributing
+
+Merlin is open source software, and we welcome contribution!
+
+We follow a gitflow - like [this](http://www.dalescott.net/using-gitflow-with-githubs-fork-pull-model/).
+To contribute, fork your own copy.
+All changes are done via pull requests from your fork into the appropriate main repository branch.
+Features and additions into `develop`, hot fixes into `master`.
+
+### Squash
+Before merging a pull request, please [squash your commits](https://github.com/LLNL/merlin/wiki/Squash-commits) down to a single concise commit message.
+
+### More...
+Complete developer documentation is [here](https://merlin.readthedocs.io/en/latest/merlin_developer.html).
+
diff --git a/README.md b/README.md
index 49cacf351..9d8be283d 100644
--- a/README.md
+++ b/README.md
@@ -1,16 +1,6 @@
-
+![alt text][logo]
+
+[logo]: https://github.com/LLNL/merlin/blob/master/docs/images/merlin.png "Merlin logo"
Welcome to the Merlin README, a condensed guide. For more in-depth Merlin
information, try our [web docs here](https://merlin.readthedocs.io/).
@@ -121,6 +111,10 @@ namely a demo workflow that has simulation and machine-learning:
More documentation on the example workflows can be found under
'Running the Examples'.
+# Code of Conduct
+Please note that Merlin has a
+[**Code of Conduct**](.github/CODE_OF_CONDUCT.md). By participating in
+the Merlin community, you agree to abide by its rules.
# Running the Examples
Example workflows can be found in the `workflows/` directory.
diff --git a/docs/source/spack.rst b/docs/source/spack.rst
index 9701c55b8..c85ea959c 100644
--- a/docs/source/spack.rst
+++ b/docs/source/spack.rst
@@ -9,16 +9,6 @@ Merlin will then be installed in this specific version allowing for
multiple python versions on a single system without the need for a
virtualenv. The py-merlinwf package builds with python3.6+.
-Checkout merlin spack repo
-**************************
-
-
-Get the latest version of merlin.spack from github.
-
-::
-
- git clone https://github.com/LLNL/merlin.spack.git
-
Checkout spack
**************
@@ -32,6 +22,16 @@ make sure merlin and spack are in separate directories.
git clone https://github.com/spack/spack.git
+The merlin addition to spack is currently in pull requests, you can clone a
+branch that has all of the PR changes here:
+
+.. code:: bash
+
+ git clone https://github.com/koning/spack.git
+ cd spack
+ git checkout feature/koning/merlinwf
+
+
Setup spack
***********
@@ -57,18 +57,12 @@ To see the compilers.
spack compiler list
-Add the merlin.spack repo to spack (done once):
-
-.. code:: bash
-
- spack repo add //merlin.spack
-
Build merlin
************
-Build merlin, this will take a *long* time, be prepared to wait. It is going
-to build python and everything python needs including numpy.
+Build merlin, this will take a *long* time, be prepared to wait. It will
+build python and all python modules merlin needs including numpy.
.. code:: bash
diff --git a/merlin/__init__.py b/merlin/__init__.py
index 31b8d970d..073551f80 100644
--- a/merlin/__init__.py
+++ b/merlin/__init__.py
@@ -37,7 +37,8 @@
import os
import sys
-__version__ = "1.0.0"
+
+__version__ = "1.0.2"
VERSION = __version__
PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "")
diff --git a/merlin/celery.py b/merlin/celery.py
index 92996ee16..59b5f4e8d 100644
--- a/merlin/celery.py
+++ b/merlin/celery.py
@@ -29,10 +29,7 @@
###############################################################################
"""Updated celery configuration."""
-from __future__ import (
- absolute_import,
- print_function,
-)
+from __future__ import absolute_import, print_function
import logging
import os
@@ -43,10 +40,7 @@
from celery.signals import worker_process_init
import merlin.common.security.encrypt_backend_traffic
-from merlin.config import (
- broker,
- results_backend,
-)
+from merlin.config import broker, results_backend
from merlin.log_formatter import FORMATS
from merlin.router import route_for_task
diff --git a/merlin/common/sample_index_factory.py b/merlin/common/sample_index_factory.py
index 288b7ace5..436dc3d60 100644
--- a/merlin/common/sample_index_factory.py
+++ b/merlin/common/sample_index_factory.py
@@ -33,10 +33,7 @@
"""
from parse import parse
-from merlin.common.sample_index import (
- MAX_SAMPLE,
- SampleIndex,
-)
+from merlin.common.sample_index import MAX_SAMPLE, SampleIndex
from merlin.utils import cd
diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py
index 7b58e3418..1d12bdab1 100644
--- a/merlin/common/tasks.py
+++ b/merlin/common/tasks.py
@@ -29,34 +29,18 @@
###############################################################################
"""Test tasks."""
-from __future__ import (
- absolute_import,
- unicode_literals,
-)
+from __future__ import absolute_import, unicode_literals
import logging
import os
-from celery import (
- chain,
- chord,
- group,
- shared_task,
- signature,
-)
-from celery.exceptions import (
- OperationalError,
- TimeoutError,
-)
+from celery import chain, chord, group, shared_task, signature
+from celery.exceptions import OperationalError, TimeoutError
from merlin.common.abstracts.enums import ReturnCode
from merlin.common.sample_index import uniform_directories
from merlin.common.sample_index_factory import create_hierarchy
-from merlin.exceptions import (
- HardFailException,
- InvalidChainException,
- RetryException,
-)
+from merlin.exceptions import HardFailException, InvalidChainException, RetryException
from merlin.router import stop_workers
from merlin.spec.expansion import (
parameter_substitutions_for_cmd,
diff --git a/merlin/display.py b/merlin/display.py
index cb4dcd0cb..25aa518f2 100644
--- a/merlin/display.py
+++ b/merlin/display.py
@@ -37,10 +37,7 @@
from tabulate import tabulate
from merlin.ascii_art import banner_small
-from merlin.config import (
- broker,
- results_backend,
-)
+from merlin.config import broker, results_backend
from merlin.config.configfile import default_config_info
diff --git a/merlin/main.py b/merlin/main.py
index d7b3f9b35..b000f8ad5 100644
--- a/merlin/main.py
+++ b/merlin/main.py
@@ -42,16 +42,10 @@
)
from contextlib import suppress
-from merlin import (
- VERSION,
- router,
-)
+from merlin import VERSION, router
from merlin.ascii_art import banner_small
from merlin.log_formatter import setup_logging
-from merlin.spec.expansion import (
- RESERVED,
- get_spec_with_expansion,
-)
+from merlin.spec.expansion import RESERVED, get_spec_with_expansion
from merlin.study.study import MerlinStudy
from merlin.utils import ARRAY_FILE_FORMATS
diff --git a/merlin/spec/expansion.py b/merlin/spec/expansion.py
index 4012f797e..803dc60ba 100644
--- a/merlin/spec/expansion.py
+++ b/merlin/spec/expansion.py
@@ -30,16 +30,10 @@
import logging
from collections import ChainMap
-from os.path import (
- expanduser,
- expandvars,
-)
+from os.path import expanduser, expandvars
from merlin.common.abstracts.enums import ReturnCode
-from merlin.spec.override import (
- dump_with_overrides,
- error_override_vars,
-)
+from merlin.spec.override import dump_with_overrides, error_override_vars
from merlin.spec.specification import MerlinSpec
diff --git a/merlin/study/celeryadapter.py b/merlin/study/celeryadapter.py
index 79868b096..4b05f5c8e 100644
--- a/merlin/study/celeryadapter.py
+++ b/merlin/study/celeryadapter.py
@@ -37,16 +37,8 @@
import time
from contextlib import suppress
-from merlin.study.batch import (
- batch_check_parallel,
- batch_worker_launch,
-)
-from merlin.utils import (
- get_procs,
- get_yaml_var,
- is_running,
- regex_list_filter,
-)
+from merlin.study.batch import batch_check_parallel, batch_worker_launch
+from merlin.utils import get_procs, get_yaml_var, is_running, regex_list_filter
LOG = logging.getLogger(__name__)
diff --git a/merlin/study/study.py b/merlin/study/study.py
index c2f3e7140..914d59064 100644
--- a/merlin/study/study.py
+++ b/merlin/study/study.py
@@ -41,14 +41,8 @@
from merlin.common.abstracts.enums import ReturnCode
from merlin.spec import defaults
-from merlin.spec.expansion import (
- determine_user_variables,
- expand_line,
-)
-from merlin.spec.override import (
- dump_with_overrides,
- error_override_vars,
-)
+from merlin.spec.expansion import determine_user_variables, expand_line
+from merlin.spec.override import dump_with_overrides, error_override_vars
from merlin.spec.specification import MerlinSpec
from merlin.study.dag import DAG
from merlin.utils import load_array_file
diff --git a/merlin/utils.py b/merlin/utils.py
index 46a1f4fe2..560ba1f3a 100644
--- a/merlin/utils.py
+++ b/merlin/utils.py
@@ -36,10 +36,7 @@
import os
import re
import subprocess
-from contextlib import (
- contextmanager,
- suppress,
-)
+from contextlib import contextmanager, suppress
from copy import deepcopy
from types import SimpleNamespace
diff --git a/requirements/dev.txt b/requirements/dev.txt
index d300b663b..bd3b12e13 100644
--- a/requirements/dev.txt
+++ b/requirements/dev.txt
@@ -4,7 +4,6 @@ dep-license
flake8
isort
pytest
-pytest-runner
sphinx
sphinx_rtd_theme
diff --git a/requirements/release.txt b/requirements/release.txt
index 08ca47770..976c32560 100644
--- a/requirements/release.txt
+++ b/requirements/release.txt
@@ -1,11 +1,12 @@
+cached_property
celery[redis]>=4.3.0
+coloredlogs
+cryptography
+importlib_resources; python_version < '3.7'
numpy
parse
psutil>=5.1.0
-cryptography
-cached_property
-coloredlogs
-importlib_resources; python_version < '3.7'
+pyyaml>=5.1.2
# Install tip of maestrowf
-e git+https://github.com/LLNL/maestrowf.git#egg=maestrowf
diff --git a/setup.py b/setup.py
index e9294f1ba..a1c98ab04 100644
--- a/setup.py
+++ b/setup.py
@@ -89,23 +89,27 @@ def extras_require():
setup(
- name="merlin",
+ name="merlinwf",
version=version,
description="The building blocks of workflows!",
long_description=readme(),
- classifiers=["Programming Language :: Python :: 3.6+"],
+ long_description_content_type="text/markdown",
+ classifiers=[
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ ],
keywords="machine learning workflow",
- url="https://lc.llnl.gov/bitbucket/projects/MLSI/repos/merlin/browse",
+ url="https://github.com/LLNL/merlin",
license="MIT",
packages=find_packages(exclude=["tests.*", "tests"]),
install_requires=install_requires(),
extras_require=extras_require(),
- setup_requires=["pytest-runner"],
- tests_require=["pytest"],
entry_points={
"console_scripts": [
"merlin=merlin.main:main",
- "merlin-templates=merlin.merlin_templates:main"
+ "merlin-templates=merlin.merlin_templates:main",
]
},
include_package_data=True,
diff --git a/tests/common/test_sample_index.py b/tests/common/test_sample_index.py
index e5d34f32b..ba18db1e8 100644
--- a/tests/common/test_sample_index.py
+++ b/tests/common/test_sample_index.py
@@ -3,10 +3,7 @@
import unittest
from contextlib import suppress
-from merlin.common.sample_index_factory import (
- create_hierarchy,
- read_hierarchy,
-)
+from merlin.common.sample_index_factory import create_hierarchy, read_hierarchy
TEST_DIR = "UNIT_TEST_SPACE"
diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py
index 96469268e..a35265866 100644
--- a/tests/integration/run_tests.py
+++ b/tests/integration/run_tests.py
@@ -40,10 +40,7 @@
from contextlib import suppress
from glob import glob
from re import search
-from subprocess import (
- PIPE,
- Popen,
-)
+from subprocess import PIPE, Popen
OUTPUT_DIR = "cli_test_studies"
@@ -343,11 +340,16 @@ def define_tests():
demo = "workflows/feature_demo/feature_demo.yaml"
simple = "workflows/simple_chain/simple_chain.yaml"
black = "black --check --target-version py36"
+ config_dir = "./CLI_TEST_MERLIN_CONFIG"
return {
"merlin": ("merlin", ReturnCodeCond(1)),
"merlin help": ("merlin --help", ReturnCodeCond()),
"merlin version": ("merlin --version", ReturnCodeCond()),
+ "merlin config": (
+ f"merlin config -o {config_dir}; rm -rf {config_dir}",
+ ReturnCodeCond(),
+ ),
"run-workers echo simple_chain": (
f"{workers} {simple} --echo",
[ReturnCodeCond(), RegexCond(celery_regex)],
@@ -447,7 +449,7 @@ def setup_argparse():
nargs="+",
default=None,
help="Provide space-delimited ids of tests you want to run."
- "Example: '--ids 1 5 8'",
+ "Example: '--ids 1 5 8 13'",
)
return parser