add pipelines to package

MANIFEST.in

@@ -0,0 +1 @@
+recursive-include brainsets_pipelines *

brainsets/cli.py

@@ -48,6 +48,8 @@ def prepare(dataset, cores):
         process = subprocess.run(
             [
                 "snakemake",
+                "-s",
+                str(Path(__file__).parent.parent / "brainsets_pipelines" / "Snakefile"),
                 "--config",
                 f"raw_dir={config['raw_dir']}",
                 f"processed_dir={config['processed_dir']}",

brainsets/utils/__init__.py

@@ -0,0 +1,4 @@
+from . import dandi_utils
+from . import dir_utils
+from . import mat_utils
+from . import split

Snakefile → brainsets_pipelines/Snakefile

@@ -17,13 +17,13 @@ config["PROCESSED_DIR"] = expand_path(f"{config['processed_dir']}")
 # include all snakefiles for all individual datasets
 # includes are relative to the directory of the Snakefile in which they occur
 module perich_miller_population_2018_module:
-    snakefile: "pipelines/perich_miller_population_2018/Snakefile"
+    snakefile: "perich_miller_population_2018/Snakefile"
     config: config
 use rule * from perich_miller_population_2018_module as perich_miller_population_2018_*
 use rule all from perich_miller_population_2018_module as perich_miller_population_2018
 
 module pei_pandarinath_nlb_2021_module:
-    snakefile: "pipelines/pei_pandarinath_nlb_2021/Snakefile"
+    snakefile: "pei_pandarinath_nlb_2021/Snakefile"
     config: config
 use rule * from pei_pandarinath_nlb_2021_module as pei_pandarinath_nlb_2021_*
 use rule all from pei_pandarinath_nlb_2021_module as pei_pandarinath_nlb_2021

pipelines/pei_pandarinath_nlb_2021/Snakefile → brainsets_pipelines/pei_pandarinath_nlb_2021/Snakefile

@@ -20,7 +20,6 @@ checkpoint pei_pandarinath_nlb_2021_download_data:
 
 rule prepare_data:
     input:
-        py_script = f"pipelines/{DATASET}/prepare_data.py",
         nwb_file = f"{RAW_DIR}/{DATASET}/{{file}}"
     output:
         temp(f"{PROCESSED_DIR}/{DATASET}/tmp/{{file}}.txt")
@@ -29,7 +28,7 @@ rule prepare_data:
     shell:
         f"""
         mkdir -p {PROCESSED_DIR}/{DATASET}/tmp
-        python {{input.py_script}} --input_file {{input.nwb_file}} --output_dir {PROCESSED_DIR}/{DATASET} >> {{log}}
+        python -m brainsets_pipelines.{DATASET}.prepare_data --input_file {{input.nwb_file}} --output_dir {PROCESSED_DIR}/{DATASET} >> {{log}}
         find {PROCESSED_DIR}/{DATASET}/ -type f -name "*.h5" | sed "s|^{PROCESSED_DIR}/{DATASET}//||" > {{output}}
         """
 

pipelines/perich_miller_population_2018/Snakefile → brainsets_pipelines/perich_miller_population_2018/Snakefile

@@ -21,7 +21,6 @@ checkpoint perich_miller_population_2018_download_data:
 
 rule prepare_data:
     input:
-        py_script = f"pipelines/{DATASET}/prepare_data.py",
         nwb_file = f"{RAW_DIR}/{DATASET}/{{file}}"
     output:
         temp(f"{PROCESSED_DIR}/{DATASET}/tmp/{{file}}.txt")
@@ -30,7 +29,7 @@ rule prepare_data:
     shell:
         f"""
         mkdir -p {PROCESSED_DIR}/{DATASET}/tmp
-        python {{input.py_script}} --input_file {{input.nwb_file}} --output_dir {PROCESSED_DIR}/{DATASET} >> {{log}}
+        python -m brainsets_pipelines.{DATASET}.prepare_data --input_file {{input.nwb_file}} --output_dir {PROCESSED_DIR}/{DATASET} >> {{log}}
         find {PROCESSED_DIR}/{DATASET}/ -type f -name "*.h5" | sed "s|^{PROCESSED_DIR}/{DATASET}//||" > {{output}}
         """
 

pyproject.toml

@@ -51,3 +51,7 @@ Documentation = "https://brainsets.readthedocs.io/en/latest/"
 
 [project.scripts]
 brainsets = "brainsets.cli:cli"
+
+[tool.setuptools]
+packages = ["brainsets", "brainsets.utils", "brainsets_pipelines"]
+include-package-data = true