Skip to content

Commit

Permalink
feat: add pipen.run() as a function to run a pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
pwwang committed Aug 3, 2024
1 parent 6733f33 commit ea1f53e
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 12 deletions.
17 changes: 9 additions & 8 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ pip install -U pipen
`example.py`

```python
from pipen import Proc, Pipen
from pipen import Proc, Pipen, run

class P1(Proc):
"""Sort input file"""
Expand All @@ -45,11 +45,12 @@ class P2(Proc):
output = "outfile:file:result.txt"
script = "paste <(seq 1 3) {{in.infile}} > {{out.outfile}}"

class MyPipeline(Pipen):
starts = P1
# class MyPipeline(Pipen):
# starts = P1

if __name__ == "__main__":
MyPipeline().run()
# MyPipeline().run()
run("MyPipeline", starts=P1)
```

```shell
Expand All @@ -64,12 +65,12 @@ if __name__ == "__main__":
06-09 23:15:29 I core _ ____/__/ / _ ____/_ /___ _ /| /
06-09 23:15:29 I core /_/ /___/ /_/ /_____/ /_/ |_/
06-09 23:15:29 I core
06-09 23:15:29 I core version: 0.14.5
06-09 23:15:29 I core version: 0.15.3
06-09 23:15:29 I core
06-09 23:15:29 I core ╔═══════════════════════════════════════════════════╗
06-09 23:15:29 I core ║ MYPIPELINE ║
06-09 23:15:29 I core ╚═══════════════════════════════════════════════════╝
06-09 23:15:29 I core plugins : verbose v0.11.0
06-09 23:15:29 I core plugins : verbose v0.12.0
06-09 23:15:29 I core # procs : 2
06-09 23:15:29 I core profile : default
06-09 23:15:29 I core outdir : /home/pwwang/github/pipen/MyPipeline-output
Expand Down Expand Up @@ -126,7 +127,8 @@ if __name__ == "__main__":
## Examples

See more examples at `examples/` and a more realcase example at:
https://github.com/pwwang/pipen-report/tree/master/example

<https://github.com/pwwang/pipen-report/tree/master/example>

## Plugin gallery

Expand All @@ -150,7 +152,6 @@ Plugins make `pipen` even better.
- [`pipen-cli-require`][24]: A pipen cli plugin check the requirements of a pipeline
- [`pipen-cli-run`][22]: A pipen cli plugin to run a process or a pipeline


[1]: https://pwwang.github.io/pipen
[2]: https://pwwang.github.io/pipen/CHANGELOG
[3]: https://pwwang.github.io/pipen/examples
Expand Down
47 changes: 46 additions & 1 deletion docs/running.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ class P2(Proc):
Pipen().set_starts(P1, P2).set_data(<data for P1>, None)
```


## Running with a different profile

`Pipen.run()` accepts an argument `profile`, which allows you to use different profile from configuration files to run the pipeline:
Expand All @@ -65,4 +64,50 @@ Pipen().run("sge")

See [configurations][1] for more details.

## Shortcut for running a pipeline

```python
import pipen

class P1(pipen.Proc):
...

class P2(pipen.Proc):
...

class P3(pipen.Proc):
requires = [P1, P2]
...

pipen.run("MyPipeline", starts=[P1, P2], data=[<data for P1>, <data for P2>])
```

```python
>>> help(pipen.run)

run(
name: 'str',
starts: 'Type[Proc] | List[Type[Proc]]',
data: 'Iterable' = None,
*,
desc: 'str' = None,
outdir: 'str | PathLike' = None,
profile: 'str' = 'default',
**kwargs,
) -> 'bool'
Shortcut to run a pipeline

Args:
name: The name of the pipeline
starts: The start processes
data: The input data for the start processes
desc: The description of the pipeline
outdir: The output directory of the results
profile: The profile to use
**kwargs: Other options pass to Pipen to create the pipeline

Returns:
True if the pipeline ends successfully else False
```

[1]: ../configurations
2 changes: 1 addition & 1 deletion pipen/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
"""A pipeline framework for python"""
from .pipen import Pipen
from .pipen import Pipen, run
from .proc import Proc
from .procgroup import ProcGroup

Expand Down
34 changes: 34 additions & 0 deletions pipen/pipen.py
Original file line number Diff line number Diff line change
Expand Up @@ -440,3 +440,37 @@ def build_proc_relationships(self) -> None:
)

self.pbar = PipelinePBar(len(self.procs), self.name.upper())


def run(
name: str,
starts: Type[Proc] | List[Type[Proc]],
data: Iterable = None,
*,
desc: str = None,
outdir: str | PathLike = None,
profile: str = "default",
**kwargs,
) -> bool:
"""Shortcut to run a pipeline
Args:
name: The name of the pipeline
starts: The start processes
data: The input data for the start processes
desc: The description of the pipeline
outdir: The output directory of the results
profile: The profile to use
**kwargs: Other options pass to Pipen to create the pipeline
Returns:
True if the pipeline ends successfully else False
"""
pipeline = Pipen(
name=name,
desc=desc,
outdir=outdir,
**kwargs,
)
pipeline.set_starts(starts).set_data(data)
return pipeline.run(profile)
18 changes: 16 additions & 2 deletions tests/test_pipen.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
from re import template
import pytest
from pipen import Proc, Pipen
from pipen import Proc, Pipen, run
from pipen.exceptions import (
PipenException,
ProcDependencyError,
PipenSetDataError,
)
Expand Down Expand Up @@ -207,3 +206,18 @@ class MyPipe4(Pipen):

with pytest.raises(PipenOrProcNameError, match="already used by another"):
MyPipe4().run()


@pytest.mark.forked
def test_run():
class RProc1(Proc):
input = "a"
output = "b:var:{{in.a}}"

class RProc2(Proc):
requires = RProc1
input = "b"
output = "c:file:{{in.b}}"
script = "touch {{out.c}}"

assert run("MyPipe", RProc1)

0 comments on commit ea1f53e

Please sign in to comment.