Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build a Pipeline abstraction #462

Merged
merged 35 commits into from
Aug 2, 2024
Merged
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
008404f
add initial draft of pipeline abstraction
mdekstrand Jul 29, 2024
f88bc1f
refactor pipeline docs slightly
mdekstrand Jul 29, 2024
c1b1b3d
add missing headers
mdekstrand Jul 29, 2024
a051adb
doc updates
mdekstrand Jul 29, 2024
f21e449
clean up some type tests & docs
mdekstrand Jul 29, 2024
e8c5231
fix doc syntax
mdekstrand Jul 29, 2024
8897711
document fallback todo
mdekstrand Jul 29, 2024
363de4f
quick scorer note
mdekstrand Jul 29, 2024
f7ce0ea
document common pipeline names
mdekstrand Jul 29, 2024
77124a8
Add the replace_component and alias methods
mdekstrand Jul 29, 2024
d89e82a
add serialization docs
mdekstrand Jul 30, 2024
953d19b
test format
mdekstrand Aug 1, 2024
b67b598
implement working typechecks
mdekstrand Aug 1, 2024
02c097e
initial tests for pipeline behavior
mdekstrand Aug 1, 2024
c6711d5
we can start to run pipelines
mdekstrand Aug 1, 2024
a5ae7d6
pipelines don't have caches rn
mdekstrand Aug 1, 2024
1a02b74
fix infinite loop in edge cases of graph topology
mdekstrand Aug 1, 2024
bf02aa3
support literal nodes
mdekstrand Aug 1, 2024
c7f6186
support defaults
mdekstrand Aug 1, 2024
5831469
lazily type-check inputs
mdekstrand Aug 1, 2024
4d8bb13
clean up component replacement tests
mdekstrand Aug 1, 2024
0bbe241
implement fallback nodes
mdekstrand Aug 1, 2024
05a29e3
fix typing import
mdekstrand Aug 2, 2024
616ff4c
fix tests for windows int width
mdekstrand Aug 2, 2024
c0ebf7f
refactor nodes into separate package
mdekstrand Aug 2, 2024
9ec5704
move pipeline into top level
mdekstrand Aug 2, 2024
6f030cc
document nodes as opaque
mdekstrand Aug 2, 2024
6e52078
update type-check logic for python 3.10
mdekstrand Aug 2, 2024
89f75ed
rewrite pipeline run for readability
mdekstrand Aug 2, 2024
60224f0
document JSON config
mdekstrand Aug 2, 2024
1e11f08
improve test coverage and test cycle detection
mdekstrand Aug 2, 2024
92340f7
cover + test fallback failure
mdekstrand Aug 2, 2024
6fb6518
get training working
mdekstrand Aug 2, 2024
06ca3b9
fix pipeline types
mdekstrand Aug 2, 2024
69be637
add 'missing' option to node method
mdekstrand Aug 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion lenskit/lenskit/pipeline/runner.py
Original file line number Diff line number Diff line change
@@ -58,6 +58,7 @@ def run(self, node: Node[Any], *, required: bool = True) -> Any:
self._run_node(node, required)
self.status[node.name] = "finished"
except Exception as e:
_log.error("node %s failed with error %s", node, e)
self.status[node.name] = "failed"
raise e

@@ -73,7 +74,7 @@ def _run_node(self, node: Node[Any], required: bool) -> None:
self._run_component(name, comp, inputs, wiring)
case FallbackNode(name, alts):
self._run_fallback(name, alts)
case _:
case _: # pragma: nocover
raise RuntimeError(f"invalid node {node}")

def _inject_input(self, name: str, types: set[type] | None, required: bool) -> None:
43 changes: 43 additions & 0 deletions lenskit/tests/test_pipeline.py
Original file line number Diff line number Diff line change
@@ -140,6 +140,31 @@ def incr(msg: str) -> str:
pipe.run(node, msg=47)


def test_component_type_mismatch():
pipe = Pipeline()

def incr(msg: str) -> str:
return msg

node = pipe.add_component("return", incr, msg=47)
with raises(TypeError):
pipe.run(node)


def test_component_unwired_input():
pipe = Pipeline()
msg = pipe.create_input("msg", str)

def ident(msg: str, m2: str | None) -> str:
if m2:
return msg + m2
else:
return msg

node = pipe.add_component("return", ident, msg=msg)
assert pipe.run(node, msg="hello") == "hello"


def test_chain():
pipe = Pipeline()
x = pipe.create_input("x", int)
@@ -183,6 +208,24 @@ def add(x: int, y: int) -> int:
assert pipe.run(nd, a=3, b=7) == 6


def test_cycle():
pipe = Pipeline()
b = pipe.create_input("b", int)

def double(x: int) -> int:
return x * 2

def add(x: int, y: int) -> int:
return x + y

nd = pipe.add_component("double", double)
na = pipe.add_component("add", add, x=nd, y=b)
pipe.connect(nd, x=na)

with raises(RuntimeError, match="cycle"):
pipe.run(a=1, b=7)


def test_replace_component():
pipe = Pipeline()
a = pipe.create_input("a", int)