refactor(engine): Wire-up basic Python protocol run logic (#8032)

Closes #7844

Co-authored-by: Sanniti Pimpley <[email protected]>

api/src/opentrons/file_runner/abstract_file_runner.py

@@ -10,9 +10,14 @@ def load(self) -> None:
         """Prepare runner and engine state prior to starting the run."""
         ...
 
+    @abstractmethod
+    async def run(self) -> None:
+        """Run the protocol file to completion."""
+        ...
+
     @abstractmethod
     def play(self) -> None:
-        """Start (or un-pause) running the protocol file."""
+        """Resume running the protocol file after a pause."""
         ...
 
     @abstractmethod

api/src/opentrons/file_runner/context_creator.py

@@ -0,0 +1,30 @@
+"""Factory for Python Protocol API instances."""
+import asyncio
+from opentrons.protocol_engine import ProtocolEngine
+from opentrons.protocol_engine.clients import SyncClient, ChildThreadTransport
+from opentrons.protocol_api_experimental import ProtocolContext
+
+
+class ContextCreator:
+    """A factory to build Python ProtocolContext instances."""
+
+    def __init__(
+        self,
+        engine: ProtocolEngine,
+        loop: asyncio.AbstractEventLoop,
+    ) -> None:
+        """Initialize the factory with access to a ProtocolEngine.
+
+        Arguments:
+            engine: ProtocolEngine instance the context should be using.
+            loop: Event loop where the ProtocolEngine is running.
+        """
+        self._engine = engine
+        self._loop = loop
+
+    def create(self) -> ProtocolContext:
+        """Create a fresh ProtocolContext wired to a ProtocolEngine."""
+        transport = ChildThreadTransport(engine=self._engine, loop=self._loop)
+        client = SyncClient(transport=transport)
+
+        return ProtocolContext(engine_client=client)

api/src/opentrons/file_runner/create_file_runner.py

@@ -1,15 +1,21 @@
 """Protocol runner factory."""
+import asyncio
 from typing import Optional
 
 from opentrons.protocol_engine import ProtocolEngine
 from opentrons.protocols.runner import CommandTranslator
 
 from .abstract_file_runner import AbstractFileRunner
+from .protocol_file import ProtocolFileType, ProtocolFile
+
 from .json_file_runner import JsonFileRunner
 from .json_file_reader import JsonFileReader
 from .command_queue_worker import CommandQueueWorker
+
 from .python_file_runner import PythonFileRunner
-from .protocol_file import ProtocolFileType, ProtocolFile
+from .python_reader import PythonFileReader
+from .context_creator import ContextCreator
+from .python_executor import PythonExecutor
 
 
 def create_file_runner(
@@ -36,6 +42,12 @@ def create_file_runner(
                 command_queue_worker=CommandQueueWorker(engine),
             )
         elif protocol_file.file_type == ProtocolFileType.PYTHON:
-            return PythonFileRunner()
+            loop = asyncio.get_running_loop()
+            return PythonFileRunner(
+                file=protocol_file,
+                file_reader=PythonFileReader(),
+                context_creator=ContextCreator(engine=engine, loop=loop),
+                executor=PythonExecutor(loop=loop),
+            )
 
     raise NotImplementedError("Other runner types not yet supported")

api/src/opentrons/file_runner/json_file_runner.py

@@ -41,8 +41,15 @@ def load(self) -> None:
         for cmd in translated_items:
             self._protocol_engine.add_command(cmd)
 
+    async def run(self) -> None:
+        """Run the protocol to completion."""
+        # TODO(mc, 2021-06-30): this will not work with pause/resume. Rework
+        # so that queue worker has `wait_for_done` method, instead.
+        self.play()
+        await self._command_queue_worker.wait_to_be_idle()
+
     def play(self) -> None:
-        """Start (or un-pause) running the JSON protocol file."""
+        """Resume running the JSON protocol file."""
         self._command_queue_worker.play()
 
     def pause(self) -> None:

api/src/opentrons/file_runner/python_executor.py

@@ -0,0 +1,38 @@
+"""Python protocol executor."""
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+from typing import Optional
+
+from opentrons.protocol_api_experimental import ProtocolContext
+from .python_reader import PythonProtocol
+
+
+class PythonExecutor:
+    """Execute a given PythonProtocol's run method with a ProtocolContext."""
+
+    def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
+        """Initialize the exector with its dependencies and a thread pool."""
+        self._loop = loop
+        self._thread_pool = ThreadPoolExecutor(max_workers=1)
+        self._protocol: Optional[PythonProtocol] = None
+        self._context: Optional[ProtocolContext] = None
+
+    def load(
+        self,
+        protocol: PythonProtocol,
+        context: ProtocolContext,
+    ) -> None:
+        """Load the executor with the Protocol and ProtocolContext."""
+        self._protocol = protocol
+        self._context = context
+
+    async def execute(self) -> None:
+        """Execute the previously loaded Protocol."""
+        assert self._protocol, "Expected PythonExecutor.load to have been called"
+        assert self._context, "Expected PythonExecutor.load to have been called"
+
+        await self._loop.run_in_executor(
+            executor=self._thread_pool,
+            func=partial(self._protocol.run, self._context),
+        )

api/src/opentrons/file_runner/python_file_runner.py

@@ -1,16 +1,39 @@
 """File runner interfaces for Python protocols."""
 from .abstract_file_runner import AbstractFileRunner
+from .protocol_file import ProtocolFile
+from .python_reader import PythonFileReader
+from .python_executor import PythonExecutor
+from .context_creator import ContextCreator
 
 
 class PythonFileRunner(AbstractFileRunner):
     """Python protocol file runner."""
 
+    def __init__(
+        self,
+        file: ProtocolFile,
+        file_reader: PythonFileReader,
+        context_creator: ContextCreator,
+        executor: PythonExecutor,
+    ) -> None:
+        """Initialize the runner with its protocol file and dependencies."""
+        self._file = file
+        self._file_reader = file_reader
+        self._context_creator = context_creator
+        self._executor = executor
+
     def load(self) -> None:
         """Prepare to run the Python protocol file."""
-        raise NotImplementedError("Python protocol loading not implemented")
+        protocol = self._file_reader.read(protocol_file=self._file)
+        context = self._context_creator.create()
+        self._executor.load(protocol=protocol, context=context)
+
+    async def run(self) -> None:
+        """Run the protocol to completion."""
+        await self._executor.execute()
 
     def play(self) -> None:
-        """Start (or un-pause) running the Python protocol file."""
+        """Resume running the Python protocol file after a pause."""
         raise NotImplementedError()
 
     def pause(self) -> None:

api/src/opentrons/file_runner/python_reader.py

@@ -0,0 +1,39 @@
+"""Python file reading and parsing."""
+import importlib.util
+from types import ModuleType
+
+from opentrons.protocol_api_experimental import ProtocolContext
+from .protocol_file import ProtocolFile
+
+
+class PythonProtocol:
+    """A thin wrapper around the imported Python module."""
+
+    def __init__(self, protocol_module: ModuleType) -> None:
+        """Wrap the passed in protocol module."""
+        self._protocol_module = protocol_module
+
+    def run(self, context: ProtocolContext) -> None:
+        """Call the protocol module's run method."""
+        return self._protocol_module.run(context)  # type: ignore[attr-defined]
+
+
+class PythonFileReader:
+    """A reader for Python protocol files.
+
+    Gets a Python protocol's metadata (TODO) and run method.
+    """
+
+    @staticmethod
+    def read(protocol_file: ProtocolFile) -> PythonProtocol:
+        """Read a Python protocol as a `import`ed Python module."""
+        # TODO(mc, 2021-06-30): better module name logic
+        spec = importlib.util.spec_from_file_location(
+            name="protocol",
+            location=protocol_file.file_path,
+        )
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)  # type: ignore[union-attr]
+
+        # TODO(mc, 2021-06-30): actually check that this module shape is good
+        return PythonProtocol(protocol_module=module)

api/tests/opentrons/data/testosaur_v2.py

@@ -1,19 +1,19 @@
 from opentrons import types
 
 metadata = {
-    'protocolName': 'Testosaur',
-    'author': 'Opentrons <[email protected]>',
-    'description': 'A variant on "Dinosaur" for testing',
-    'source': 'Opentrons Repository',
-    'apiLevel': '2.0',
+    "protocolName": "Testosaur",
+    "author": "Opentrons <[email protected]>",
+    "description": 'A variant on "Dinosaur" for testing',
+    "source": "Opentrons Repository",
+    "apiLevel": "2.0",
 }
 
 
 def run(ctx):
     ctx.home()
-    tr = ctx.load_labware('opentrons_96_tiprack_300ul', 1)
-    right = ctx.load_instrument('p300_single', types.Mount.RIGHT, [tr])
-    lw = ctx.load_labware('corning_96_wellplate_360ul_flat', 2)
+    tr = ctx.load_labware("opentrons_96_tiprack_300ul", 1)
+    right = ctx.load_instrument("p300_single", types.Mount.RIGHT, [tr])
+    lw = ctx.load_labware("corning_96_wellplate_360ul_flat", 2)
     right.pick_up_tip()
     right.aspirate(10, lw.wells()[0].bottom())
     right.dispense(10, lw.wells()[1].bottom())

api/tests/opentrons/data/testosaur_v3.py

@@ -0,0 +1,24 @@
+from opentrons import types
+
+metadata = {
+    "protocolName": "Testosaur Version 3",
+    "author": "Opentrons <[email protected]>",
+    "description": 'A variant on "Dinosaur" for testing with Protocol API v3',
+    "source": "Opentrons Repository",
+    "apiLevel": "3.0",
+}
+
+
+def run(ctx):
+    tip_rack = ctx.load_labware("opentrons_96_tiprack_300ul", 8)
+    source = ctx.load_labware("nest_12_reservoir_15ml", 1)
+    dest = ctx.load_labware("corning_96_wellplate_360ul_flat", 2)
+
+    pipette = ctx.load_instrument("p300_single_gen2", types.Mount.RIGHT, [])
+
+    for i in range(4):
+        pipette.pick_up_tip(tip_rack.well(i))
+        pipette.pick_up_tip(tip_rack.well(i))
+        pipette.aspirate(50, source.wells_by_name()["A1"])
+        pipette.dispense(50, dest.well(i))
+        pipette.drop_tip(tip_rack.well(i))

api/tests/opentrons/file_runner/smoke_tests/__init__.py

@@ -0,0 +1 @@
+"""Smoke tests for the protocol file runners."""

api/tests/opentrons/file_runner/smoke_tests/test_python.py

@@ -0,0 +1,85 @@
+"""Test running a Python protocol on the ProtocolEngine."""
+import pytest
+import textwrap
+from pathlib import Path
+from decoy import matchers
+
+from opentrons_shared_data.pipette.dev_types import LabwareUri
+from opentrons.hardware_control import API as HardwareAPI
+from opentrons.protocol_api_experimental import DeckSlotName
+
+from opentrons.protocol_engine import (
+    ProtocolEngine,
+    LabwareData,
+    DeckSlotLocation,
+    create_protocol_engine,
+)
+
+from opentrons.file_runner import (
+    create_file_runner,
+    AbstractFileRunner,
+    ProtocolFile,
+    ProtocolFileType,
+)
+
+
+@pytest.fixture
+def python_protocol_file(tmp_path: Path) -> ProtocolFile:
+    """Get an on-disk, minimal Python protocol fixture."""
+    file_path = tmp_path / "protocol-name.py"
+    file_path.write_text(
+        textwrap.dedent(
+            """
+            # my protocol
+            metadata = {
+                "apiLevel": "3.0",
+            }
+            def run(ctx):
+                ctx.load_labware(
+                    load_name="opentrons_96_tiprack_300ul",
+                    location="1",
+                )
+            """
+        ),
+        encoding="utf-8",
+    )
+
+    return ProtocolFile(file_path=file_path, file_type=ProtocolFileType.PYTHON)
+
+
+@pytest.fixture
+async def engine(hardware: HardwareAPI) -> ProtocolEngine:
+    """Get a real ProtocolEngine hooked into a simulating HardwareAPI."""
+    return await create_protocol_engine(hardware=hardware)
+
+
+@pytest.fixture
+async def subject(
+    python_protocol_file: ProtocolFile,
+    engine: ProtocolEngine,
+) -> AbstractFileRunner:
+    """Get a real file runner."""
+    return create_file_runner(
+        protocol_file=python_protocol_file,
+        engine=engine,
+    )
+
+
+async def test_python_protocol_runner(
+    engine: ProtocolEngine,
+    subject: AbstractFileRunner,
+) -> None:
+    """It should run a Python protocol on the ProtocolEngine."""
+    subject.load()
+    await subject.run()
+
+    expected_labware_entry = (
+        matchers.IsA(str),
+        LabwareData(
+            location=DeckSlotLocation(slot=DeckSlotName.SLOT_1),
+            uri=LabwareUri("opentrons/opentrons_96_tiprack_300ul/1"),
+            calibration=(matchers.IsA(float), matchers.IsA(float), matchers.IsA(float)),
+        ),
+    )
+
+    assert expected_labware_entry in engine.state_view.labware.get_all_labware()