From 32a93e61af4e38f727e89974bffa0a812dd5e26a Mon Sep 17 00:00:00 2001 From: Mehdi Azabou Date: Tue, 26 Nov 2024 12:16:01 -0500 Subject: [PATCH] add pipelines to package --- MANIFEST.in | 1 + brainsets/cli.py | 2 + brainsets/utils/__init__.py | 4 ++ Snakefile => brainsets_pipelines/Snakefile | 4 +- .../pei_pandarinath_nlb_2021/Snakefile | 3 +- .../pei_pandarinath_nlb_2021/prepare_data.py | 0 .../perich_miller_population_2018/README.md | 0 .../perich_miller_population_2018/Snakefile | 3 +- .../prepare_data.py | 0 pyproject.toml | 4 ++ setup.py | 67 ------------------- 11 files changed, 15 insertions(+), 73 deletions(-) create mode 100644 MANIFEST.in create mode 100644 brainsets/utils/__init__.py rename Snakefile => brainsets_pipelines/Snakefile (88%) rename {pipelines => brainsets_pipelines}/pei_pandarinath_nlb_2021/Snakefile (90%) rename {pipelines => brainsets_pipelines}/pei_pandarinath_nlb_2021/prepare_data.py (100%) rename {pipelines => brainsets_pipelines}/perich_miller_population_2018/README.md (100%) rename {pipelines => brainsets_pipelines}/perich_miller_population_2018/Snakefile (90%) rename {pipelines => brainsets_pipelines}/perich_miller_population_2018/prepare_data.py (100%) delete mode 100644 setup.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..47cb675 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +recursive-include brainsets_pipelines * diff --git a/brainsets/cli.py b/brainsets/cli.py index 32d43d1..29fda05 100644 --- a/brainsets/cli.py +++ b/brainsets/cli.py @@ -48,6 +48,8 @@ def prepare(dataset, cores): process = subprocess.run( [ "snakemake", + "-s", + str(Path(__file__).parent.parent / "brainsets_pipelines" / "Snakefile"), "--config", f"raw_dir={config['raw_dir']}", f"processed_dir={config['processed_dir']}", diff --git a/brainsets/utils/__init__.py b/brainsets/utils/__init__.py new file mode 100644 index 0000000..3aadc4e --- /dev/null +++ b/brainsets/utils/__init__.py @@ -0,0 +1,4 @@ +from . import dandi_utils +from . import dir_utils +from . import mat_utils +from . import split diff --git a/Snakefile b/brainsets_pipelines/Snakefile similarity index 88% rename from Snakefile rename to brainsets_pipelines/Snakefile index 9d7facc..1ce93e5 100644 --- a/Snakefile +++ b/brainsets_pipelines/Snakefile @@ -17,13 +17,13 @@ config["PROCESSED_DIR"] = expand_path(f"{config['processed_dir']}") # include all snakefiles for all individual datasets # includes are relative to the directory of the Snakefile in which they occur module perich_miller_population_2018_module: - snakefile: "pipelines/perich_miller_population_2018/Snakefile" + snakefile: "perich_miller_population_2018/Snakefile" config: config use rule * from perich_miller_population_2018_module as perich_miller_population_2018_* use rule all from perich_miller_population_2018_module as perich_miller_population_2018 module pei_pandarinath_nlb_2021_module: - snakefile: "pipelines/pei_pandarinath_nlb_2021/Snakefile" + snakefile: "pei_pandarinath_nlb_2021/Snakefile" config: config use rule * from pei_pandarinath_nlb_2021_module as pei_pandarinath_nlb_2021_* use rule all from pei_pandarinath_nlb_2021_module as pei_pandarinath_nlb_2021 diff --git a/pipelines/pei_pandarinath_nlb_2021/Snakefile b/brainsets_pipelines/pei_pandarinath_nlb_2021/Snakefile similarity index 90% rename from pipelines/pei_pandarinath_nlb_2021/Snakefile rename to brainsets_pipelines/pei_pandarinath_nlb_2021/Snakefile index 78604c1..61a06a2 100644 --- a/pipelines/pei_pandarinath_nlb_2021/Snakefile +++ b/brainsets_pipelines/pei_pandarinath_nlb_2021/Snakefile @@ -20,7 +20,6 @@ checkpoint pei_pandarinath_nlb_2021_download_data: rule prepare_data: input: - py_script = f"pipelines/{DATASET}/prepare_data.py", nwb_file = f"{RAW_DIR}/{DATASET}/{{file}}" output: temp(f"{PROCESSED_DIR}/{DATASET}/tmp/{{file}}.txt") @@ -29,7 +28,7 @@ rule prepare_data: shell: f""" mkdir -p {PROCESSED_DIR}/{DATASET}/tmp - python {{input.py_script}} --input_file {{input.nwb_file}} --output_dir {PROCESSED_DIR}/{DATASET} >> {{log}} + python -m brainsets_pipelines.{DATASET}.prepare_data --input_file {{input.nwb_file}} --output_dir {PROCESSED_DIR}/{DATASET} >> {{log}} find {PROCESSED_DIR}/{DATASET}/ -type f -name "*.h5" | sed "s|^{PROCESSED_DIR}/{DATASET}//||" > {{output}} """ diff --git a/pipelines/pei_pandarinath_nlb_2021/prepare_data.py b/brainsets_pipelines/pei_pandarinath_nlb_2021/prepare_data.py similarity index 100% rename from pipelines/pei_pandarinath_nlb_2021/prepare_data.py rename to brainsets_pipelines/pei_pandarinath_nlb_2021/prepare_data.py diff --git a/pipelines/perich_miller_population_2018/README.md b/brainsets_pipelines/perich_miller_population_2018/README.md similarity index 100% rename from pipelines/perich_miller_population_2018/README.md rename to brainsets_pipelines/perich_miller_population_2018/README.md diff --git a/pipelines/perich_miller_population_2018/Snakefile b/brainsets_pipelines/perich_miller_population_2018/Snakefile similarity index 90% rename from pipelines/perich_miller_population_2018/Snakefile rename to brainsets_pipelines/perich_miller_population_2018/Snakefile index 14eca66..4e21952 100644 --- a/pipelines/perich_miller_population_2018/Snakefile +++ b/brainsets_pipelines/perich_miller_population_2018/Snakefile @@ -21,7 +21,6 @@ checkpoint perich_miller_population_2018_download_data: rule prepare_data: input: - py_script = f"pipelines/{DATASET}/prepare_data.py", nwb_file = f"{RAW_DIR}/{DATASET}/{{file}}" output: temp(f"{PROCESSED_DIR}/{DATASET}/tmp/{{file}}.txt") @@ -30,7 +29,7 @@ rule prepare_data: shell: f""" mkdir -p {PROCESSED_DIR}/{DATASET}/tmp - python {{input.py_script}} --input_file {{input.nwb_file}} --output_dir {PROCESSED_DIR}/{DATASET} >> {{log}} + python -m brainsets_pipelines.{DATASET}.prepare_data --input_file {{input.nwb_file}} --output_dir {PROCESSED_DIR}/{DATASET} >> {{log}} find {PROCESSED_DIR}/{DATASET}/ -type f -name "*.h5" | sed "s|^{PROCESSED_DIR}/{DATASET}//||" > {{output}} """ diff --git a/pipelines/perich_miller_population_2018/prepare_data.py b/brainsets_pipelines/perich_miller_population_2018/prepare_data.py similarity index 100% rename from pipelines/perich_miller_population_2018/prepare_data.py rename to brainsets_pipelines/perich_miller_population_2018/prepare_data.py diff --git a/pyproject.toml b/pyproject.toml index 94345cf..6d53c1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,3 +51,7 @@ Documentation = "https://brainsets.readthedocs.io/en/latest/" [project.scripts] brainsets = "brainsets.cli:cli" + +[tool.setuptools] +packages = ["brainsets", "brainsets.utils", "brainsets_pipelines"] +include-package-data = true diff --git a/setup.py b/setup.py deleted file mode 100644 index d231ee3..0000000 --- a/setup.py +++ /dev/null @@ -1,67 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name="brainsets", - version="0.1.0", - author="Mehdi Azabou", - author_email="mehdiazabou@gmail.com", - description="A package for processing neural datasets", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - packages=find_packages(), - install_requires=[ - "temporaldata", - "scipy~=1.10.1", - "pynwb~=2.2.0", - "setuptools~=60.2.0", - "numpy~=1.23.5", - "pandas~=1.5.3", - "jsonschema~=4.21.1", - "scikit-image~=0.19.3", - "tqdm~=4.64.1", - "rich==13.3.2", - "msgpack~=1.0.5", - "snakemake~=7.32.3", - "pydantic~=2.0", - "pulp==2.7.0", - "click~=8.1.3", - "dandi==0.61.2", - ], - extras_require={ - "dev": [ - "pytest~=7.2.1", - "black==24.2.0", - "pre-commit>=3.5.0", - "flake8", - ], - "eeg": [ - "mne", - ], - "zenodo": [ - "zenodo-get~=1.5.1", - ], - "allen": [ - "allensdk==2.16.2", - ], - "all": [ - "pytest~=7.2.1", - "black==24.2.0", - "pre-commit>=3.5.0", - "flake8", - "zenodo-get~=1.5.1", - "allensdk==2.16.2", - "mne", - ], - }, - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - python_requires=">=3.8", - entry_points={ - "console_scripts": [ - "brainsets=brainsets.cli:cli", - ], - }, -)