diff --git a/.travis.yml b/.travis.yml index 5d1fb6d56..8a9b9ecfc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,7 @@ install: - "pip install --upgrade pip" - "pip install cryptography" - "pip install -r requirements.txt" + - "pip install -r workflows/feature_demo/requirements.txt" - "pip install -e ." - "pip install -r requirements/mysql.txt" - "pip install --upgrade sphinx" diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ab9f3a68..4ab16232a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,26 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.4] - 2019-12-04 + +### Added +- `requirements.txt` files and `scripts` directories for internal workflow examples. + +### Fixed +- Added missing dependency `tabulate` to `release.txt`. + +## [1.0.3] - 2019-12-04 + +### Added +Added the requirements files to the MANIFEST.in file for source +distributions. + +## [1.0.2] - 2019-12-04 +Negligible changes related to PyPI. + +## [1.0.1] - 2019-12-04 +Negligible changes related to PyPI. + ## [1.0.0] - 2019-11-19 First public release. See the docs and merlin -h for details. Here are some highlights. diff --git a/MANIFEST.in b/MANIFEST.in index b2f4f33b6..3a1b2a29d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,5 @@ prune docs prune tests include workflows/* include merlin/data/* +include requirements.txt +include requirements/* diff --git a/merlin/__init__.py b/merlin/__init__.py index 073551f80..2db64ffd4 100644 --- a/merlin/__init__.py +++ b/merlin/__init__.py @@ -38,7 +38,7 @@ import sys -__version__ = "1.0.2" +__version__ = "1.0.4" VERSION = __version__ PATH_TO_PROJ = os.path.join(os.path.dirname(__file__), "") diff --git a/requirements/release.txt b/requirements/release.txt index 976c32560..8ca46abc8 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -3,10 +3,12 @@ celery[redis]>=4.3.0 coloredlogs cryptography importlib_resources; python_version < '3.7' +# maestrowf>=1.1.5 numpy parse psutil>=5.1.0 pyyaml>=5.1.2 +tabulate # Install tip of maestrowf -e git+https://github.com/LLNL/maestrowf.git#egg=maestrowf diff --git a/workflows/feature_demo/feature_demo.yaml b/workflows/feature_demo/feature_demo.yaml index 1377feadd..e8e1b0944 100644 --- a/workflows/feature_demo/feature_demo.yaml +++ b/workflows/feature_demo/feature_demo.yaml @@ -12,11 +12,12 @@ env: WORKER_NAME: demo_worker VERIFY_QUEUE: default_verify_queue - HELLO: $(SPECROOT)/hello_world.py - FEATURES: $(SPECROOT)/features.json - TRANSLATE: $(SPECROOT)/translator.py - COLLECT: $(SPECROOT)/collector.py - MAKE_SAMPLES: $(SPECROOT)/make_samples.py + SCRIPTS: $(SPECROOT)/scripts + HELLO: $(SCRIPTS)/hello_world.py + FEATURES: $(SCRIPTS)/features.json + TRANSLATE: $(SCRIPTS)/translator.py + COLLECT: $(SCRIPTS)/collector.py + MAKE_SAMPLES: $(SCRIPTS)/make_samples.py SHARED: $(SPECROOT)/../shared LEARN: $(SHARED)/learn.py @@ -137,7 +138,7 @@ merlin: args: -l INFO --concurrency 3 --prefetch-multiplier 1 -Ofair samples: generate: - cmd: python $(SPECROOT)/make_samples.py -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + cmd: python $(MAKE_SAMPLES) -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy # can be a file glob of numpy sample files. file: $(MERLIN_INFO)/samples.npy column_labels: [X0, X1] diff --git a/workflows/feature_demo/requirements.txt b/workflows/feature_demo/requirements.txt new file mode 100644 index 000000000..0c62c11b4 --- /dev/null +++ b/workflows/feature_demo/requirements.txt @@ -0,0 +1 @@ +sklearn diff --git a/workflows/feature_demo/collector.py b/workflows/feature_demo/scripts/collector.py similarity index 100% rename from workflows/feature_demo/collector.py rename to workflows/feature_demo/scripts/collector.py diff --git a/workflows/feature_demo/features.json b/workflows/feature_demo/scripts/features.json similarity index 100% rename from workflows/feature_demo/features.json rename to workflows/feature_demo/scripts/features.json diff --git a/workflows/feature_demo/hello_world.py b/workflows/feature_demo/scripts/hello_world.py similarity index 100% rename from workflows/feature_demo/hello_world.py rename to workflows/feature_demo/scripts/hello_world.py diff --git a/workflows/feature_demo/make_samples.py b/workflows/feature_demo/scripts/make_samples.py similarity index 100% rename from workflows/feature_demo/make_samples.py rename to workflows/feature_demo/scripts/make_samples.py diff --git a/workflows/feature_demo/translator.py b/workflows/feature_demo/scripts/translator.py similarity index 100% rename from workflows/feature_demo/translator.py rename to workflows/feature_demo/scripts/translator.py diff --git a/workflows/flux/flux_par.yaml b/workflows/flux/flux_par.yaml index 3562e97bc..eda977d2c 100644 --- a/workflows/flux/flux_par.yaml +++ b/workflows/flux/flux_par.yaml @@ -16,7 +16,7 @@ study: - description: Build the code name: build run: - cmd: mpicc -o mpi_hello $(SPECROOT)/hello.c >& build.out + cmd: mpicc -o mpi_hello $(SPECROOT)/scripts/hello.c >& build.out task_queue: flux_par - description: Echo the params name: runs @@ -60,4 +60,4 @@ merlin: column_labels: [V1, V2] file: $(MERLIN_INFO)/samples.npy generate: - cmd: python $(SPECROOT)/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + cmd: python $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy diff --git a/workflows/flux/paper.yaml b/workflows/flux/paper.yaml index c7e984c3b..4b554d75a 100644 --- a/workflows/flux/paper.yaml +++ b/workflows/flux/paper.yaml @@ -19,7 +19,7 @@ study: name: build run: cmd: | - mpicc -o mpi_hello_sleep $(SPECROOT)/hello_sleep.c >& build.out + mpicc -o mpi_hello_sleep $(SPECROOT)/scripts/hello_sleep.c >& build.out # Add start time to flux flux wreck setenv FLUX_START_TIME=${FLUX_START_TIME} flux wreck setenv FLUX_START_SECONDS=${FLUX_START_SECONDS} @@ -47,7 +47,7 @@ study: run: cmd: | #export PATH=/collab/usr/gapps/merlin/spack/idesign/${SYS_TYPE}/bin:$PATH - $(SPECROOT)/flux_info.py >flux_timings.out + $(SPECROOT)/scripts/flux_info.py >flux_timings.out depends: [runs*] task_queue: flux_paper @@ -68,4 +68,4 @@ merlin: column_labels: [V1, V2] file: $(MERLIN_INFO)/samples.npy generate: - cmd: python $(SPECROOT)/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + cmd: python $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy diff --git a/workflows/flux/requirements.txt b/workflows/flux/requirements.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/workflows/flux/requirements.txt @@ -0,0 +1 @@ + diff --git a/workflows/flux/flux_info.py b/workflows/flux/scripts/flux_info.py similarity index 100% rename from workflows/flux/flux_info.py rename to workflows/flux/scripts/flux_info.py diff --git a/workflows/flux/hello.c b/workflows/flux/scripts/hello.c similarity index 100% rename from workflows/flux/hello.c rename to workflows/flux/scripts/hello.c diff --git a/workflows/flux/hello_sleep.c b/workflows/flux/scripts/hello_sleep.c similarity index 100% rename from workflows/flux/hello_sleep.c rename to workflows/flux/scripts/hello_sleep.c diff --git a/workflows/flux/make_samples.py b/workflows/flux/scripts/make_samples.py similarity index 100% rename from workflows/flux/make_samples.py rename to workflows/flux/scripts/make_samples.py diff --git a/workflows/flux/paper_workers.sbatch b/workflows/flux/scripts/paper_workers.sbatch similarity index 100% rename from workflows/flux/paper_workers.sbatch rename to workflows/flux/scripts/paper_workers.sbatch diff --git a/workflows/flux/test_workers.sbatch b/workflows/flux/scripts/test_workers.sbatch similarity index 100% rename from workflows/flux/test_workers.sbatch rename to workflows/flux/scripts/test_workers.sbatch diff --git a/workflows/flux/workers.bsub b/workflows/flux/scripts/workers.bsub similarity index 100% rename from workflows/flux/workers.bsub rename to workflows/flux/scripts/workers.bsub diff --git a/workflows/flux/workers.sbatch b/workflows/flux/scripts/workers.sbatch similarity index 100% rename from workflows/flux/workers.sbatch rename to workflows/flux/scripts/workers.sbatch diff --git a/workflows/slurm/requirements.txt b/workflows/slurm/requirements.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/workflows/slurm/requirements.txt @@ -0,0 +1 @@ + diff --git a/workflows/slurm/hello.c b/workflows/slurm/scripts/hello.c similarity index 100% rename from workflows/slurm/hello.c rename to workflows/slurm/scripts/hello.c diff --git a/workflows/slurm/make_samples.py b/workflows/slurm/scripts/make_samples.py similarity index 100% rename from workflows/slurm/make_samples.py rename to workflows/slurm/scripts/make_samples.py diff --git a/workflows/slurm/test_workers.sbatch b/workflows/slurm/scripts/test_workers.sbatch similarity index 100% rename from workflows/slurm/test_workers.sbatch rename to workflows/slurm/scripts/test_workers.sbatch diff --git a/workflows/slurm/workers.sbatch b/workflows/slurm/scripts/workers.sbatch similarity index 100% rename from workflows/slurm/workers.sbatch rename to workflows/slurm/scripts/workers.sbatch diff --git a/workflows/slurm/slurm_par.yaml b/workflows/slurm/slurm_par.yaml index 720046091..893e1c7a3 100644 --- a/workflows/slurm/slurm_par.yaml +++ b/workflows/slurm/slurm_par.yaml @@ -13,7 +13,7 @@ study: - description: Build the code name: build run: - cmd: mpicc -o mpi_hello $(SPECROOT)/hello.c >& build.out + cmd: mpicc -o mpi_hello $(SPECROOT)/scripts/hello.c >& build.out task_queue: slurm_par - description: Echo the params name: runs @@ -46,4 +46,4 @@ merlin: column_labels: [V1, V2] file: $(MERLIN_INFO)/samples.npy generate: - cmd: python $(SPECROOT)/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + cmd: python $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy diff --git a/workflows/slurm/slurm_test.yaml b/workflows/slurm/slurm_test.yaml index 1c783bc6a..dcb5bb598 100644 --- a/workflows/slurm/slurm_test.yaml +++ b/workflows/slurm/slurm_test.yaml @@ -40,4 +40,4 @@ merlin: column_labels: [V1, V2] file: $(MERLIN_INFO)/samples.npy generate: - cmd: python $(SPECROOT)/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy + cmd: python $(SPECROOT)/scripts/make_samples.py -dims 2 -n $(N_SAMPLES) -outfile=$(MERLIN_INFO)/samples.npy