From 266c41a1bde9048269d323f7f3268ba4835640e5 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Thu, 14 Oct 2021 12:29:56 -0400 Subject: [PATCH 01/19] Module control refactors --- api/src/opentrons/hardware_control/api.py | 8 ++- .../opentrons/hardware_control/controller.py | 3 +- .../hardware_control/module_control.py | 54 ++++++++++--------- .../opentrons/hardware_control/simulator.py | 2 - 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/api/src/opentrons/hardware_control/api.py b/api/src/opentrons/hardware_control/api.py index 89f30cf80bd..3ce5a0317a5 100644 --- a/api/src/opentrons/hardware_control/api.py +++ b/api/src/opentrons/hardware_control/api.py @@ -214,7 +214,9 @@ async def blink(): api_instance = cls(backend, loop=checked_loop, config=checked_config) await api_instance.cache_instruments() - module_controls = await AttachedModulesControl.build(api_instance) + module_controls = await AttachedModulesControl.build( + api_instance, board_revision=backend.board_revision + ) backend.module_controls = module_controls checked_loop.create_task(backend.watch(loop=checked_loop)) backend.start_gpio_door_watcher( @@ -260,7 +262,9 @@ async def build_hardware_simulator( ) api_instance = cls(backend, loop=checked_loop, config=checked_config) await api_instance.cache_instruments() - module_controls = await AttachedModulesControl.build(api_instance) + module_controls = await AttachedModulesControl.build( + api_instance, board_revision=backend.board_revision + ) backend.module_controls = module_controls await backend.watch() return api_instance diff --git a/api/src/opentrons/hardware_control/controller.py b/api/src/opentrons/hardware_control/controller.py index 909ffaf5e64..e63be888731 100644 --- a/api/src/opentrons/hardware_control/controller.py +++ b/api/src/opentrons/hardware_control/controller.py @@ -11,7 +11,7 @@ aionotify = None from opentrons.drivers.smoothie_drivers import SmoothieDriver -from opentrons.drivers.rpi_drivers import build_gpio_chardev, usb +from opentrons.drivers.rpi_drivers import build_gpio_chardev import opentrons.config from opentrons.config import pipette_config from opentrons.config.types import RobotConfig @@ -77,7 +77,6 @@ def __init__(self, config: RobotConfig, gpio: GPIODriverLike): config=self.config, gpio_chardev=self._gpio_chardev ) self._cached_fw_version: Optional[str] = None - self._usb = usb.USBBus(self._board_revision) self._module_controls: Optional[AttachedModulesControl] = None try: self._event_watcher = self._build_event_watcher() diff --git a/api/src/opentrons/hardware_control/module_control.py b/api/src/opentrons/hardware_control/module_control.py index 9b63b70bfc0..a9d9b7d6917 100644 --- a/api/src/opentrons/hardware_control/module_control.py +++ b/api/src/opentrons/hardware_control/module_control.py @@ -1,3 +1,4 @@ +from __future__ import annotations import logging import asyncio import os @@ -6,11 +7,10 @@ from glob import glob from opentrons.config import IS_ROBOT, IS_LINUX -from opentrons.drivers.rpi_drivers import types +from opentrons.drivers.rpi_drivers import types, usb, usb_simulator from opentrons.hardware_control.modules import ModuleAtPort -from .execution_manager import ExecutionManager -from .types import AionotifyEvent +from .types import AionotifyEvent, BoardRevision from . import modules @@ -25,13 +25,20 @@ class AttachedModulesControl: USB port information and finally building a module object. """ - def __init__(self, api): + def __init__(self, api, board_revision: BoardRevision) -> None: self._available_modules: List[modules.AbstractModule] = [] self._api = api + self._usb = ( + usb.USBBus(board_revision) + if not api.is_simulator + else usb_simulator.USBBusSimulator(board_revision) + ) @classmethod - async def build(cls, api_instance): - mc_instance = cls(api_instance) + async def build( + cls, api_instance, board_revision: BoardRevision + ) -> AttachedModulesControl: + mc_instance = cls(api_instance, board_revision) if not api_instance.is_simulator: await mc_instance.register_modules(mc_instance.scan()) return mc_instance @@ -40,10 +47,6 @@ async def build(cls, api_instance): def available_modules(self) -> List[modules.AbstractModule]: return self._available_modules - @property - def api(self): - return self._api - async def build_module( self, port: str, @@ -56,9 +59,9 @@ async def build_module( port=port, usb_port=usb_port, which=model, - simulating=self.api.is_simulator, + simulating=self._api.is_simulator, loop=loop, - execution_manager=self.api._execution_manager, + execution_manager=self._api._execution_manager, sim_model=sim_model, ) @@ -107,14 +110,15 @@ async def register_modules( # destroy removed mods await self.unregister_modules(removed_mods_at_ports) - sorted_mods_at_port = self.api._backend._usb.match_virtual_ports( - new_mods_at_ports - ) + sorted_mods_at_port = self._usb.match_virtual_ports(new_mods_at_ports) # build new mods for mod in sorted_mods_at_port: new_instance = await self.build_module( - port=mod.port, usb_port=mod.usb_port, model=mod.name, loop=self.api.loop + port=mod.port, + usb_port=mod.usb_port, + model=mod.name, + loop=self._api.loop, ) self._available_modules.append(new_instance) log.info( @@ -145,7 +149,7 @@ async def parse_modules( for module in self.available_modules: if mod_type == module.name(): matching_modules.append(module) - if self.api.is_simulator: + if self._api.is_simulator: module_builder = { "magdeck": modules.MagDeck.build, "tempdeck": modules.TempDeck.build, @@ -154,10 +158,10 @@ async def parse_modules( if module_builder: simulating_module = await module_builder( port="", - usb_port=self.api._backend._usb.find_port(""), + usb_port=self._usb.find_port(""), simulating=True, - loop=self.api.loop, - execution_manager=ExecutionManager(loop=self.api.loop), + loop=self._api.loop, + execution_manager=self._api._execution_manager, sim_model=by_model.value, ) simulated_module = simulating_module @@ -212,14 +216,16 @@ def get_module_at_port(port: str) -> Optional[modules.ModuleAtPort]: return modules.ModuleAtPort(port=f"/dev/{port}", name=name) return None - async def handle_module_appearance(self, event: AionotifyEvent): + async def handle_module_appearance(self, event: AionotifyEvent) -> None: """Only called upon availability of aionotify. Check that the file system has changed and either remove or add modules depending on the result. - :param event_name: The title of the even passed into aionotify. - :param event_flags: AionotifyFlags dataclass that maps flags listed from - the aionotify event. + Args: + event: The event passed from aionotify. + + Returns: + None """ maybe_module_at_port = self.get_module_at_port(event.name) new_modules = None diff --git a/api/src/opentrons/hardware_control/simulator.py b/api/src/opentrons/hardware_control/simulator.py index 8b07b263f5b..684ab0e5bd1 100644 --- a/api/src/opentrons/hardware_control/simulator.py +++ b/api/src/opentrons/hardware_control/simulator.py @@ -14,7 +14,6 @@ from opentrons.drivers.smoothie_drivers import SimulatingDriver from opentrons.drivers.rpi_drivers.gpio_simulator import SimulatingGPIOCharDev -from opentrons.drivers.rpi_drivers import usb_simulator from . import modules from .types import BoardRevision, Axis @@ -154,7 +153,6 @@ def _sanitize_attached_instrument( self._log = MODULE_LOG.getChild(repr(self)) self._strict_attached = bool(strict_attached_instruments) self._board_revision = BoardRevision.OG - self._usb = usb_simulator.USBBusSimulator(self._board_revision) # TODO (lc 05-12-2021) In a follow-up refactor that pulls the execution # manager responsbility into the controller/backend itself as opposed # to the hardware api controller. From f7c03798d1e98775c02ed19a93c34714fa31b200 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Fri, 22 Oct 2021 15:08:19 -0400 Subject: [PATCH 02/19] new package. --- .../emulation/module_server.py | 216 ------------------ .../emulation/module_server/__init__.py | 8 + .../emulation/module_server/client.py | 64 ++++++ .../emulation/module_server/helpers.py | 48 ++++ .../emulation/module_server/models.py | 20 ++ .../emulation/module_server/server.py | 100 ++++++++ 6 files changed, 240 insertions(+), 216 deletions(-) delete mode 100644 api/src/opentrons/hardware_control/emulation/module_server.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/__init__.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/client.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/helpers.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/models.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/server.py diff --git a/api/src/opentrons/hardware_control/emulation/module_server.py b/api/src/opentrons/hardware_control/emulation/module_server.py deleted file mode 100644 index 2a09a21afd2..00000000000 --- a/api/src/opentrons/hardware_control/emulation/module_server.py +++ /dev/null @@ -1,216 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging - -from opentrons.hardware_control.emulation.types import ModuleType -from typing_extensions import Literal, Final -from typing import Dict, List, Set, Sequence, Optional -from pydantic import BaseModel - -from opentrons.hardware_control.emulation.proxy import ProxyListener -from opentrons.hardware_control.emulation.settings import ModuleServerSettings - - -log = logging.getLogger(__name__) - -MessageDelimiter: Final = b"\n" - - -class ModuleStatusServer(ProxyListener): - """Server notifying of module connections.""" - - def __init__(self, settings: ModuleServerSettings) -> None: - """Constructor - - Args: - settings: app settings - """ - self._settings = settings - self._connections: Dict[str, Connection] = {} - self._clients: Set[asyncio.StreamWriter] = set() - - def on_server_connected( - self, server_type: str, client_uri: str, identifier: str - ) -> None: - """Called when a new module has connected. - - Args: - server_type: the type of module - client_uri: the url string for a driver to connect to - identifier: unique id for connection - - Returns: None - - """ - log.info(f"On connected {server_type} {client_uri} {identifier}") - connection = Connection( - module_type=server_type, url=client_uri, identifier=identifier - ) - self._connections[identifier] = connection - for c in self._clients: - c.write( - Message(status="connected", connections=[connection]).json().encode() - ) - c.write(b"\n") - - def on_server_disconnected(self, identifier: str) -> None: - """Called when a module has disconnected. - - Args: - identifier: unique id for the connection - - Returns: None - """ - log.info(f"On disconnected {identifier}") - try: - connection = self._connections[identifier] - del self._connections[identifier] - for c in self._clients: - c.write( - Message(status="disconnected", connections=[connection]) - .json() - .encode() - ) - c.write(MessageDelimiter) - except KeyError: - log.exception("Failed to find identifier") - - async def run(self) -> None: - """Run the server.""" - server = await asyncio.start_server( - self._handle_connection, host=self._settings.host, port=self._settings.port - ) - await server.serve_forever() - - async def _handle_connection( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter - ) -> None: - """Handle a client connection to the server.""" - log.info("Client connected to module server.") - - m = Message(status="dump", connections=list(self._connections.values())) - - writer.write(m.json().encode()) - writer.write(MessageDelimiter) - - self._clients.add(writer) - - while True: - if b"" == await reader.read(): - self._clients.remove(writer) - break - - log.info("Client disconnected from module server.") - - -class Connection(BaseModel): - """Model a single module connection.""" - - url: str - module_type: str - identifier: str - - -class Message(BaseModel): - """A message sent to module server clients.""" - - status: Literal["connected", "disconnected", "dump"] - connections: List[Connection] - - -class ModuleServerClient: - """A module server client.""" - - def __init__( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter - ) -> None: - """Constructor.""" - self._reader = reader - self._writer = writer - - @classmethod - async def connect( - cls, - host: str, - port: int, - retries: int = 3, - interval_seconds: float = 0.1, - ) -> ModuleServerClient: - """Connect to the module server. - - Args: - host: module server host. - port: module server port. - retries: number of retries - interval_seconds: time between retries. - - Returns: - None - Raises: - IOError on retry expiry. - """ - r: Optional[asyncio.StreamReader] = None - w: Optional[asyncio.StreamWriter] = None - for i in range(retries): - try: - r, w = await asyncio.open_connection(host=host, port=port) - except OSError: - await asyncio.sleep(interval_seconds) - - if r is not None and w is not None: - return ModuleServerClient(reader=r, writer=w) - else: - raise IOError( - f"Failed to connect to module_server at after {retries} retries." - ) - - async def read(self) -> Message: - """Read a message from the module server.""" - b = await self._reader.readuntil(MessageDelimiter) - m: Message = Message.parse_raw(b) - return m - - def close(self) -> None: - """Close the client.""" - self._writer.close() - - -async def wait_emulators( - client: ModuleServerClient, - modules: Sequence[ModuleType], - timeout: float, -) -> None: - """Wait for module emulators to connect. - - Args: - client: module server client. - modules: collection of of module types to wait for. - timeout: how long to wait for emulators to connect (in seconds) - - Returns: - None - Raises: - asyncio.TimeoutError on timeout. - """ - - async def _wait_modules(cl: ModuleServerClient, module_set: Set[str]) -> None: - """Read messages from module server waiting for modules in module_set to - be connected.""" - while module_set: - m: Message = await cl.read() - if m.status == "dump" or m.status == "connected": - for c in m.connections: - if c.module_type in module_set: - module_set.remove(c.module_type) - elif m.status == "disconnected": - for c in m.connections: - if c.module_type in module_set: - module_set.add(c.module_type) - - log.debug(f"after message: {m}, awaiting module set is: {module_set}") - - await asyncio.wait_for( - _wait_modules(cl=client, module_set=set(n.value for n in modules)), - timeout=timeout, - ) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/__init__.py b/api/src/opentrons/hardware_control/emulation/module_server/__init__.py new file mode 100644 index 00000000000..fd9f7229550 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/__init__.py @@ -0,0 +1,8 @@ +from .server import ModuleStatusServer +from .client import ModuleServerClient + + +__all__ = [ + "ModuleStatusServer", + "ModuleServerClient", +] diff --git a/api/src/opentrons/hardware_control/emulation/module_server/client.py b/api/src/opentrons/hardware_control/emulation/module_server/client.py new file mode 100644 index 00000000000..2881623890c --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/client.py @@ -0,0 +1,64 @@ +from __future__ import annotations +import asyncio +from typing import Optional + +from opentrons.hardware_control.emulation.module_server.models import Message +from opentrons.hardware_control.emulation.module_server.server import \ + MessageDelimiter + + +class ModuleServerClient: + """A module server client.""" + + def __init__( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """Constructor.""" + self._reader = reader + self._writer = writer + + @classmethod + async def connect( + cls, + host: str, + port: int, + retries: int = 3, + interval_seconds: float = 0.1, + ) -> ModuleServerClient: + """Connect to the module server. + + Args: + host: module server host. + port: module server port. + retries: number of retries + interval_seconds: time between retries. + + Returns: + None + Raises: + IOError on retry expiry. + """ + r: Optional[asyncio.StreamReader] = None + w: Optional[asyncio.StreamWriter] = None + for i in range(retries): + try: + r, w = await asyncio.open_connection(host=host, port=port) + except OSError: + await asyncio.sleep(interval_seconds) + + if r is not None and w is not None: + return ModuleServerClient(reader=r, writer=w) + else: + raise IOError( + f"Failed to connect to module_server at after {retries} retries." + ) + + async def read(self) -> Message: + """Read a message from the module server.""" + b = await self._reader.readuntil(MessageDelimiter) + m: Message = Message.parse_raw(b) + return m + + def close(self) -> None: + """Close the client.""" + self._writer.close() diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py new file mode 100644 index 00000000000..802a1b80c54 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -0,0 +1,48 @@ +import asyncio +from typing import Sequence, Set + +from opentrons.hardware_control.emulation.module_server.client import \ + ModuleServerClient +from opentrons.hardware_control.emulation.module_server.models import Message +from opentrons.hardware_control.emulation.module_server.server import log +from opentrons.hardware_control.emulation.types import ModuleType + + +async def wait_emulators( + client: ModuleServerClient, + modules: Sequence[ModuleType], + timeout: float, +) -> None: + """Wait for module emulators to connect. + + Args: + client: module server client. + modules: collection of of module types to wait for. + timeout: how long to wait for emulators to connect (in seconds) + + Returns: + None + Raises: + asyncio.TimeoutError on timeout. + """ + + async def _wait_modules(cl: ModuleServerClient, module_set: Set[str]) -> None: + """Read messages from module server waiting for modules in module_set to + be connected.""" + while module_set: + m: Message = await cl.read() + if m.status == "dump" or m.status == "connected": + for c in m.connections: + if c.module_type in module_set: + module_set.remove(c.module_type) + elif m.status == "disconnected": + for c in m.connections: + if c.module_type in module_set: + module_set.add(c.module_type) + + log.debug(f"after message: {m}, awaiting module set is: {module_set}") + + await asyncio.wait_for( + _wait_modules(cl=client, module_set=set(n.value for n in modules)), + timeout=timeout, + ) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/models.py b/api/src/opentrons/hardware_control/emulation/module_server/models.py new file mode 100644 index 00000000000..9ee203ea42a --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/models.py @@ -0,0 +1,20 @@ +from typing import List + +from typing_extensions import Literal + +from pydantic import BaseModel + + +class Connection(BaseModel): + """Model a single module connection.""" + + url: str + module_type: str + identifier: str + + +class Message(BaseModel): + """A message sent to module server clients.""" + + status: Literal["connected", "disconnected", "dump"] + connections: List[Connection] \ No newline at end of file diff --git a/api/src/opentrons/hardware_control/emulation/module_server/server.py b/api/src/opentrons/hardware_control/emulation/module_server/server.py new file mode 100644 index 00000000000..62d7743a39a --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/server.py @@ -0,0 +1,100 @@ +import asyncio +import logging +from typing import Dict, Set + +from opentrons.hardware_control.emulation.module_server.models import Connection, \ + Message +from opentrons.hardware_control.emulation.proxy import ProxyListener +from opentrons.hardware_control.emulation.settings import ModuleServerSettings +from typing_extensions import Final + +log = logging.getLogger(__name__) + +MessageDelimiter: Final = b"\n" + + +class ModuleStatusServer(ProxyListener): + """Server notifying of module connections.""" + + def __init__(self, settings: ModuleServerSettings) -> None: + """Constructor + + Args: + settings: app settings + """ + self._settings = settings + self._connections: Dict[str, Connection] = {} + self._clients: Set[asyncio.StreamWriter] = set() + + def on_server_connected( + self, server_type: str, client_uri: str, identifier: str + ) -> None: + """Called when a new module has connected. + + Args: + server_type: the type of module + client_uri: the url string for a driver to connect to + identifier: unique id for connection + + Returns: None + + """ + log.info(f"On connected {server_type} {client_uri} {identifier}") + connection = Connection( + module_type=server_type, url=client_uri, identifier=identifier + ) + self._connections[identifier] = connection + for c in self._clients: + c.write( + Message(status="connected", connections=[connection]).json().encode() + ) + c.write(b"\n") + + def on_server_disconnected(self, identifier: str) -> None: + """Called when a module has disconnected. + + Args: + identifier: unique id for the connection + + Returns: None + """ + log.info(f"On disconnected {identifier}") + try: + connection = self._connections[identifier] + del self._connections[identifier] + for c in self._clients: + c.write( + Message(status="disconnected", connections=[connection]) + .json() + .encode() + ) + c.write(MessageDelimiter) + except KeyError: + log.exception("Failed to find identifier") + + async def run(self) -> None: + """Run the server.""" + server = await asyncio.start_server( + self._handle_connection, host=self._settings.host, port=self._settings.port + ) + await server.serve_forever() + + async def _handle_connection( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """Handle a client connection to the server.""" + log.info("Client connected to module server.") + + m = Message(status="dump", connections=list(self._connections.values())) + + writer.write(m.json().encode()) + writer.write(MessageDelimiter) + + self._clients.add(writer) + + while True: + if b"" == await reader.read(): + self._clients.remove(writer) + break + + log.info("Client disconnected from module server.") From 15d64f65ddede272709d168301c4c2dd148ea74f Mon Sep 17 00:00:00 2001 From: amit lissack Date: Fri, 22 Oct 2021 15:51:13 -0400 Subject: [PATCH 03/19] start module control integration. --- .../emulation/module_server/helpers.py | 34 ++++++++++++++++++- .../emulation/module_server/__init__.py | 0 .../emulation/module_server/test_helpers.py | 9 +++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 api/tests/opentrons/hardware_control/emulation/module_server/__init__.py create mode 100644 api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index 802a1b80c54..279bee34fe2 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -1,11 +1,43 @@ import asyncio -from typing import Sequence, Set +from typing import Sequence, Set, Callable, List, Coroutine, Awaitable from opentrons.hardware_control.emulation.module_server.client import \ ModuleServerClient from opentrons.hardware_control.emulation.module_server.models import Message from opentrons.hardware_control.emulation.module_server.server import log from opentrons.hardware_control.emulation.types import ModuleType +from opentrons.hardware_control.modules import ModuleAtPort + +NotifyMethod = Callable[[List[ModuleAtPort], List[ModuleAtPort]], Awaitable[None]] +"""Signature of method to be notified of new and removed modules.""" + + +class ModuleListener: + """Provide a callback for listening for new and removed module connections.""" + + def __init__(self, client: ModuleServerClient, notify_method: NotifyMethod) -> None: + """Constructor. + + Args: + client: A module server client + notify_method: callback method. + + Returns: + None + """ + self._client = client + self._notify_method = notify_method + + async def run(self) -> None: + """""" + while True: + m = await self._client.read() + await self.message_to_notify(message=m, notify_method=self._notify_method) + + @staticmethod + async def message_to_notify(message: Message, notify_method: NotifyMethod) -> None: + await notify_method([], []) + async def wait_emulators( diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/__init__.py b/api/tests/opentrons/hardware_control/emulation/module_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py new file mode 100644 index 00000000000..f50c9f9194c --- /dev/null +++ b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py @@ -0,0 +1,9 @@ +import pytest +from mock import AsyncMock +from opentrons.hardware_control.emulation.module_server import helpers + + +@pytest + +async def test_message_to_notify(): + pass \ No newline at end of file From a5a2926368660147cfbe1a8aed303bf47511fc4c Mon Sep 17 00:00:00 2001 From: amit lissack Date: Fri, 22 Oct 2021 16:30:09 -0400 Subject: [PATCH 04/19] tests for module control integration. --- .../emulation/module_server/client.py | 3 +- .../emulation/module_server/helpers.py | 6 +- .../emulation/module_server/models.py | 2 +- .../emulation/module_server/server.py | 6 +- .../emulation/module_server/test_helpers.py | 140 +++++++++++++++++- 5 files changed, 145 insertions(+), 12 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/client.py b/api/src/opentrons/hardware_control/emulation/module_server/client.py index 2881623890c..bf60164a1f9 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/client.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/client.py @@ -3,8 +3,7 @@ from typing import Optional from opentrons.hardware_control.emulation.module_server.models import Message -from opentrons.hardware_control.emulation.module_server.server import \ - MessageDelimiter +from opentrons.hardware_control.emulation.module_server.server import MessageDelimiter class ModuleServerClient: diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index 279bee34fe2..450bf8ad1d3 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -1,8 +1,7 @@ import asyncio -from typing import Sequence, Set, Callable, List, Coroutine, Awaitable +from typing import Sequence, Set, Callable, List, Awaitable -from opentrons.hardware_control.emulation.module_server.client import \ - ModuleServerClient +from opentrons.hardware_control.emulation.module_server.client import ModuleServerClient from opentrons.hardware_control.emulation.module_server.models import Message from opentrons.hardware_control.emulation.module_server.server import log from opentrons.hardware_control.emulation.types import ModuleType @@ -39,7 +38,6 @@ async def message_to_notify(message: Message, notify_method: NotifyMethod) -> No await notify_method([], []) - async def wait_emulators( client: ModuleServerClient, modules: Sequence[ModuleType], diff --git a/api/src/opentrons/hardware_control/emulation/module_server/models.py b/api/src/opentrons/hardware_control/emulation/module_server/models.py index 9ee203ea42a..f2c4b0c598e 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/models.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/models.py @@ -17,4 +17,4 @@ class Message(BaseModel): """A message sent to module server clients.""" status: Literal["connected", "disconnected", "dump"] - connections: List[Connection] \ No newline at end of file + connections: List[Connection] diff --git a/api/src/opentrons/hardware_control/emulation/module_server/server.py b/api/src/opentrons/hardware_control/emulation/module_server/server.py index 62d7743a39a..13da6410392 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/server.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/server.py @@ -2,8 +2,10 @@ import logging from typing import Dict, Set -from opentrons.hardware_control.emulation.module_server.models import Connection, \ - Message +from opentrons.hardware_control.emulation.module_server.models import ( + Connection, + Message, +) from opentrons.hardware_control.emulation.proxy import ProxyListener from opentrons.hardware_control.emulation.settings import ModuleServerSettings from typing_extensions import Final diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py index f50c9f9194c..45ad51b6a68 100644 --- a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py +++ b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py @@ -1,9 +1,143 @@ +from typing import List + import pytest from mock import AsyncMock +from opentrons.drivers.rpi_drivers.types import USBPort from opentrons.hardware_control.emulation.module_server import helpers +from opentrons.hardware_control.emulation.module_server import models +from opentrons.hardware_control.modules import ModuleAtPort + + +@pytest.fixture +def mock_callback() -> AsyncMock: + """Callback mock.""" + return AsyncMock(spec=helpers.NotifyMethod) + + +@pytest.fixture +def connections() -> List[models.Connection]: + """Connection models.""" + return [ + models.Connection( + url=f"url{i}", module_type=f"module_type{i}", identifier=f"identifier{i}" + ) + for i in range(5) + ] + + +@pytest.fixture +def modules_at_port() -> List[ModuleAtPort]: + """Connection models.""" + return [ + ModuleAtPort( + port=f"url{i}", + name=f"module_type{i}", + usb_port=USBPort(name=f"identifier{i}", sub_names=[]), + ) + for i in range(5) + ] + + +async def test_message_to_notify_connected_empty(mock_callback: AsyncMock) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=[]) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with([], []) + + +async def test_message_to_notify_connected_one( + mock_callback: AsyncMock, + connections: List[models.Connection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=connections[:1]) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with(modules_at_port[:1], []) + + +async def test_message_to_notify_connected_many( + mock_callback: AsyncMock, + connections: List[models.Connection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=connections) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with(modules_at_port, []) + + +async def test_message_to_notify_disconnected_empty(mock_callback: AsyncMock) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=[]) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with([], []) + + +async def test_message_to_notify_disconnected_one( + mock_callback: AsyncMock, + connections: List[models.Connection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=connections[:1]) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with([], modules_at_port[:1]) + + +async def test_message_to_notify_disconnected_many( + mock_callback: AsyncMock, + connections: List[models.Connection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=connections) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with([], modules_at_port) + + +async def test_message_to_notify_dump_empty(mock_callback: AsyncMock) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=[]) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with([], []) + +async def test_message_to_notify_dump_one( + mock_callback: AsyncMock, + connections: List[models.Connection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=connections[:1]) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with(modules_at_port[:1], []) -@pytest -async def test_message_to_notify(): - pass \ No newline at end of file +async def test_message_to_notify_dump_many( + mock_callback: AsyncMock, + connections: List[models.Connection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=connections) + await helpers.ModuleListener.message_to_notify( + message=message, notify_method=mock_callback + ) + mock_callback.assert_called_once_with(modules_at_port, []) From 2f9600e4abfaa434c183025c6961944ff0083bb3 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Sat, 23 Oct 2021 09:32:49 -0400 Subject: [PATCH 05/19] implement connection listener. --- .../emulation/module_server/helpers.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index 450bf8ad1d3..153a67b4dc4 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -1,6 +1,7 @@ import asyncio from typing import Sequence, Set, Callable, List, Awaitable +from opentrons.drivers.rpi_drivers.types import USBPort from opentrons.hardware_control.emulation.module_server.client import ModuleServerClient from opentrons.hardware_control.emulation.module_server.models import Message from opentrons.hardware_control.emulation.module_server.server import log @@ -28,14 +29,34 @@ def __init__(self, client: ModuleServerClient, notify_method: NotifyMethod) -> N self._notify_method = notify_method async def run(self) -> None: - """""" + """Run the listener.""" while True: m = await self._client.read() await self.message_to_notify(message=m, notify_method=self._notify_method) @staticmethod async def message_to_notify(message: Message, notify_method: NotifyMethod) -> None: - await notify_method([], []) + """Call callback with results of message. + + Args: + message: Message object from module server + notify_method: callback method + + Returns: + None + """ + connections = [ + ModuleAtPort( + port=c.url, + name=c.module_type, + usb_port=USBPort(name=c.identifier, sub_names=[]), + ) + for c in message.connections + ] + if message.status == "connected" or message.status == "dump": + await notify_method(connections, []) + elif message.status == "disconnected": + await notify_method([], connections) async def wait_emulators( From 00714675c70da9632f3cf1277af6e998624e570e Mon Sep 17 00:00:00 2001 From: amit lissack Date: Sat, 23 Oct 2021 10:15:55 -0400 Subject: [PATCH 06/19] module server integrated into module control. --- .../emulation/module_server/helpers.py | 12 ++++++++++++ .../hardware_control/emulation/proxy.py | 2 +- .../hardware_control/emulation/types.py | 4 ++-- .../hardware_control/module_control.py | 19 ++++--------------- 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index 153a67b4dc4..c2ec0da562b 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -5,6 +5,7 @@ from opentrons.hardware_control.emulation.module_server.client import ModuleServerClient from opentrons.hardware_control.emulation.module_server.models import Message from opentrons.hardware_control.emulation.module_server.server import log +from opentrons.hardware_control.emulation.settings import Settings from opentrons.hardware_control.emulation.types import ModuleType from opentrons.hardware_control.modules import ModuleAtPort @@ -12,6 +13,17 @@ """Signature of method to be notified of new and removed modules.""" +async def listen_module_connection(callback: NotifyMethod) -> None: + """Listen for module emulator connections.""" + settings = Settings() + try: + client = await ModuleServerClient.connect(host=settings.module_server.host, port=settings.module_server.port, interval_seconds=1.0) + listener = ModuleListener(client=client, notify_method=callback) + await listener.run() + except IOError: + log.exception("Failed to connect to module server.") + + class ModuleListener: """Provide a callback for listening for new and removed module connections.""" diff --git a/api/src/opentrons/hardware_control/emulation/proxy.py b/api/src/opentrons/hardware_control/emulation/proxy.py index 6877c89ebf5..3ebc94a961a 100644 --- a/api/src/opentrons/hardware_control/emulation/proxy.py +++ b/api/src/opentrons/hardware_control/emulation/proxy.py @@ -115,7 +115,7 @@ async def _handle_server_connection( self._cons.append(connection) self._event_listener.on_server_connected( server_type=self._name, - client_uri=f"{socket.gethostname()}:{self._settings.driver_port}", + client_uri=f"socket://{socket.gethostname()}:{self._settings.driver_port}", identifier=connection.identifier, ) diff --git a/api/src/opentrons/hardware_control/emulation/types.py b/api/src/opentrons/hardware_control/emulation/types.py index cbdd3eecb11..1a5a7028ef3 100644 --- a/api/src/opentrons/hardware_control/emulation/types.py +++ b/api/src/opentrons/hardware_control/emulation/types.py @@ -4,7 +4,7 @@ class ModuleType(str, Enum): """Module type enumeration.""" - Magnetic = "magnetic" - Temperature = "temperature" + Magnetic = "magdeck" + Temperature = "tempdeck" Thermocycler = "thermocycler" Heatershaker = "heatershaker" diff --git a/api/src/opentrons/hardware_control/module_control.py b/api/src/opentrons/hardware_control/module_control.py index a9d9b7d6917..db79d51c2b7 100644 --- a/api/src/opentrons/hardware_control/module_control.py +++ b/api/src/opentrons/hardware_control/module_control.py @@ -8,6 +8,8 @@ from opentrons.config import IS_ROBOT, IS_LINUX from opentrons.drivers.rpi_drivers import types, usb, usb_simulator +from opentrons.hardware_control.emulation.module_server.helpers import \ + listen_module_connection from opentrons.hardware_control.modules import ModuleAtPort from .types import AionotifyEvent, BoardRevision @@ -41,6 +43,8 @@ async def build( mc_instance = cls(api_instance, board_revision) if not api_instance.is_simulator: await mc_instance.register_modules(mc_instance.scan()) + api_instance.loop.create_task(listen_module_connection(mc_instance.register_modules)) + return mc_instance @property @@ -184,21 +188,6 @@ def scan(self) -> List[modules.ModuleAtPort]: if module_at_port: discovered_modules.append(module_at_port) - # Check for emulator environment variables - emulator_uri = os.environ.get("OT_THERMOCYCLER_EMULATOR_URI") - if emulator_uri: - discovered_modules.append( - ModuleAtPort(port=emulator_uri, name="thermocycler") - ) - - emulator_uri = os.environ.get("OT_TEMPERATURE_EMULATOR_URI") - if emulator_uri: - discovered_modules.append(ModuleAtPort(port=emulator_uri, name="tempdeck")) - - emulator_uri = os.environ.get("OT_MAGNETIC_EMULATOR_URI") - if emulator_uri: - discovered_modules.append(ModuleAtPort(port=emulator_uri, name="magdeck")) - log.debug("Discovered modules: {}".format(discovered_modules)) return discovered_modules From 1383db7d360eb31d32799a12f08660378f0d9a01 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Sat, 23 Oct 2021 10:29:24 -0400 Subject: [PATCH 07/19] lint --- .../hardware_control/emulation/module_server/helpers.py | 6 +++++- api/src/opentrons/hardware_control/module_control.py | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index c2ec0da562b..12256d1ddb7 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -17,7 +17,11 @@ async def listen_module_connection(callback: NotifyMethod) -> None: """Listen for module emulator connections.""" settings = Settings() try: - client = await ModuleServerClient.connect(host=settings.module_server.host, port=settings.module_server.port, interval_seconds=1.0) + client = await ModuleServerClient.connect( + host=settings.module_server.host, + port=settings.module_server.port, + interval_seconds=1.0, + ) listener = ModuleListener(client=client, notify_method=callback) await listener.run() except IOError: diff --git a/api/src/opentrons/hardware_control/module_control.py b/api/src/opentrons/hardware_control/module_control.py index db79d51c2b7..68f0a8c2b82 100644 --- a/api/src/opentrons/hardware_control/module_control.py +++ b/api/src/opentrons/hardware_control/module_control.py @@ -8,8 +8,9 @@ from opentrons.config import IS_ROBOT, IS_LINUX from opentrons.drivers.rpi_drivers import types, usb, usb_simulator -from opentrons.hardware_control.emulation.module_server.helpers import \ - listen_module_connection +from opentrons.hardware_control.emulation.module_server.helpers import ( + listen_module_connection, +) from opentrons.hardware_control.modules import ModuleAtPort from .types import AionotifyEvent, BoardRevision @@ -43,7 +44,9 @@ async def build( mc_instance = cls(api_instance, board_revision) if not api_instance.is_simulator: await mc_instance.register_modules(mc_instance.scan()) - api_instance.loop.create_task(listen_module_connection(mc_instance.register_modules)) + api_instance.loop.create_task( + listen_module_connection(mc_instance.register_modules) + ) return mc_instance From 6cd877d9b011382c59787a4a62ea9d9dcc51d39c Mon Sep 17 00:00:00 2001 From: amit lissack Date: Sat, 23 Oct 2021 10:53:57 -0400 Subject: [PATCH 08/19] g-code-testing --- .../g_code_parsing/g_code_engine.py | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index 36c323cb617..961db324d79 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -12,7 +12,10 @@ from contextlib import contextmanager from opentrons.protocol_api import ProtocolContext from opentrons.config.robot_configs import build_config -from opentrons.hardware_control.emulation import module_server +from opentrons.hardware_control.emulation.module_server.helpers import ( + wait_emulators, + ModuleServerClient, +) from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control import API, ThreadManager from g_code_parsing.g_code_program.g_code_program import ( @@ -45,7 +48,6 @@ class GCodeEngine: def __init__(self, emulator_settings: Settings) -> None: self._config = emulator_settings - self._set_env_vars(emulator_settings) @staticmethod def _get_loop() -> asyncio.AbstractEventLoop: @@ -57,23 +59,10 @@ def _get_loop() -> asyncio.AbstractEventLoop: asyncio.set_event_loop(_loop) return asyncio.get_event_loop() - @staticmethod - def _set_env_vars(settings: Settings) -> None: - """Set URLs of where to find modules and config for smoothie""" - os.environ["OT_MAGNETIC_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.magdeck_proxy.driver_port - ) - os.environ["OT_THERMOCYCLER_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.thermocycler_proxy.driver_port - ) - os.environ["OT_TEMPERATURE_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.temperature_proxy.driver_port - ) - @staticmethod def _start_emulation_app(emulator_settings: Settings) -> Process: """Start emulated OT-2""" - modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] + modules = [ModuleType.Thermocycler, ModuleType.Magnetic, ModuleType.Temperature] def runit(): asyncio.run( @@ -85,10 +74,10 @@ def runit(): proc.start() async def _wait_ready() -> None: - c = await module_server.ModuleServerClient.connect( + c = await ModuleServerClient.connect( host="localhost", port=emulator_settings.module_server.port ) - await module_server.wait_emulators(client=c, modules=modules, timeout=5) + await wait_emulators(client=c, modules=modules, timeout=5) c.close() proc2 = Process(target=lambda: asyncio.run(_wait_ready())) From 6d867a3206701909cbfd58fbb10ac3615f436917 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Sat, 23 Oct 2021 12:49:08 -0400 Subject: [PATCH 09/19] redo docker compose to deal with separate emulator apps. --- docker-compose.yml | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b66e1846fd4..931d70f49d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,12 @@ services: build: . command: python3 -m opentrons.hardware_control.emulation.app ports: + - '8888:8888' + - '9000:9000' + - '9002:9002' + - '9003:9003' + - '9004:9004' + - '9995:9995' - '9996:9996' - '9997:9997' - '9998:9998' @@ -17,14 +23,33 @@ services: ports: - '31950:31950' environment: - - OT_API_CONFIG_DIR=/config - - OT_SMOOTHIE_EMULATOR_URI=socket://emulator:9996 - - OT_THERMOCYCLER_EMULATOR_URI=socket://emulator:9997 - - OT_TEMPERATURE_EMULATOR_URI=socket://emulator:9998 - - OT_MAGNETIC_EMULATOR_URI=socket://emulator:9999 + OT_API_CONFIG_DIR: /config + OT_SMOOTHIE_EMULATOR_URI: socket://emulator:9996 + OT_EMULATOR_module_server: '{"host": "emulator"}' links: - 'emulator' depends_on: - 'emulator' volumes: - .opentrons_config:/config:rw + tempdeck: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator tempdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + thermocycler: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator thermocycler emulator + links: + - 'emulator' + depends_on: + - 'emulator' + magdeck: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator magdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' From df7b7c90bc7771e9c73503b6f0a9ce5b3d764ef1 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Sun, 24 Oct 2021 20:23:54 -0400 Subject: [PATCH 10/19] usb port. --- .../emulation/module_server/client.py | 1 + .../emulation/module_server/helpers.py | 19 +++-- .../hardware_control/emulation/settings.py | 2 +- .../hardware_control/module_control.py | 2 +- .../emulation/module_server/test_helpers.py | 77 ++++++++++++------- .../g_code_parsing/g_code_engine.py | 16 ++-- 6 files changed, 73 insertions(+), 44 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/client.py b/api/src/opentrons/hardware_control/emulation/module_server/client.py index bf60164a1f9..f0f6652e6fa 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/client.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/client.py @@ -42,6 +42,7 @@ async def connect( for i in range(retries): try: r, w = await asyncio.open_connection(host=host, port=port) + break except OSError: await asyncio.sleep(interval_seconds) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index 12256d1ddb7..4f92f8d79be 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -43,36 +43,41 @@ def __init__(self, client: ModuleServerClient, notify_method: NotifyMethod) -> N """ self._client = client self._notify_method = notify_method + self._hub_index = 1 async def run(self) -> None: """Run the listener.""" while True: m = await self._client.read() - await self.message_to_notify(message=m, notify_method=self._notify_method) + await self.handle_message(message=m) - @staticmethod - async def message_to_notify(message: Message, notify_method: NotifyMethod) -> None: + async def handle_message(self, message: Message) -> None: """Call callback with results of message. Args: message: Message object from module server - notify_method: callback method Returns: None """ + def _next_index() -> int: + index = self._hub_index + self._hub_index += 1 + return index + connections = [ ModuleAtPort( port=c.url, name=c.module_type, - usb_port=USBPort(name=c.identifier, sub_names=[]), + usb_port=USBPort(name=c.identifier, sub_names=[], hub=_next_index()), ) for c in message.connections ] if message.status == "connected" or message.status == "dump": - await notify_method(connections, []) + await self._notify_method(connections, []) elif message.status == "disconnected": - await notify_method([], connections) + await self._notify_method([], connections) + async def wait_emulators( diff --git a/api/src/opentrons/hardware_control/emulation/settings.py b/api/src/opentrons/hardware_control/emulation/settings.py index 8e62cfc1192..192847393a4 100644 --- a/api/src/opentrons/hardware_control/emulation/settings.py +++ b/api/src/opentrons/hardware_control/emulation/settings.py @@ -29,7 +29,7 @@ class ModuleServerSettings(BaseModel): """Settings for the module server""" host: str = "0.0.0.0" - port: int = 8888 + port: int = 8989 class Settings(BaseSettings): diff --git a/api/src/opentrons/hardware_control/module_control.py b/api/src/opentrons/hardware_control/module_control.py index 68f0a8c2b82..46291779d05 100644 --- a/api/src/opentrons/hardware_control/module_control.py +++ b/api/src/opentrons/hardware_control/module_control.py @@ -33,7 +33,7 @@ def __init__(self, api, board_revision: BoardRevision) -> None: self._api = api self._usb = ( usb.USBBus(board_revision) - if not api.is_simulator + if not api.is_simulator and IS_ROBOT else usb_simulator.USBBusSimulator(board_revision) ) diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py index 45ad51b6a68..d6a00d94207 100644 --- a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py +++ b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py @@ -3,7 +3,8 @@ import pytest from mock import AsyncMock from opentrons.drivers.rpi_drivers.types import USBPort -from opentrons.hardware_control.emulation.module_server import helpers +from opentrons.hardware_control.emulation.module_server import helpers, \ + ModuleServerClient from opentrons.hardware_control.emulation.module_server import models from opentrons.hardware_control.modules import ModuleAtPort @@ -14,6 +15,18 @@ def mock_callback() -> AsyncMock: return AsyncMock(spec=helpers.NotifyMethod) +@pytest.fixture +def mock_client() -> AsyncMock: + """Mock client.""" + return AsyncMock(spec=ModuleServerClient) + + +@pytest.fixture +def subject(mock_callback: AsyncMock, mock_client: AsyncMock) -> helpers.ModuleListener: + """Test subject.""" + return helpers.ModuleListener(client=mock_client, notify_method=mock_callback) + + @pytest.fixture def connections() -> List[models.Connection]: """Connection models.""" @@ -32,112 +45,118 @@ def modules_at_port() -> List[ModuleAtPort]: ModuleAtPort( port=f"url{i}", name=f"module_type{i}", - usb_port=USBPort(name=f"identifier{i}", sub_names=[]), + usb_port=USBPort(name=f"identifier{i}", sub_names=[], hub=i), ) for i in range(5) ] -async def test_message_to_notify_connected_empty(mock_callback: AsyncMock) -> None: +async def test_handle_message_connected_empty(subject: helpers.ModuleListener, mock_callback: AsyncMock) -> None: """It should call the call back with the correct modules to add.""" message = models.Message(status="connected", connections=[]) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with([], []) -async def test_message_to_notify_connected_one( +async def test_handle_message_connected_one( + subject: helpers.ModuleListener, mock_callback: AsyncMock, connections: List[models.Connection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to add.""" message = models.Message(status="connected", connections=connections[:1]) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with(modules_at_port[:1], []) -async def test_message_to_notify_connected_many( +async def test_handle_message_connected_many( + subject: helpers.ModuleListener, mock_callback: AsyncMock, connections: List[models.Connection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to add.""" message = models.Message(status="connected", connections=connections) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with(modules_at_port, []) -async def test_message_to_notify_disconnected_empty(mock_callback: AsyncMock) -> None: +async def test_handle_message_disconnected_empty(subject: helpers.ModuleListener, mock_callback: AsyncMock) -> None: """It should call the call back with the correct modules to remove.""" message = models.Message(status="disconnected", connections=[]) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with([], []) -async def test_message_to_notify_disconnected_one( +async def test_handle_message_disconnected_one( + subject: helpers.ModuleListener, mock_callback: AsyncMock, connections: List[models.Connection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to remove.""" message = models.Message(status="disconnected", connections=connections[:1]) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with([], modules_at_port[:1]) -async def test_message_to_notify_disconnected_many( +async def test_handle_message_disconnected_many( + subject: helpers.ModuleListener, mock_callback: AsyncMock, connections: List[models.Connection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to remove.""" message = models.Message(status="disconnected", connections=connections) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with([], modules_at_port) -async def test_message_to_notify_dump_empty(mock_callback: AsyncMock) -> None: +async def test_handle_message_dump_empty(subject: helpers.ModuleListener, mock_callback: AsyncMock) -> None: """It should call the call back with the correct modules to load.""" message = models.Message(status="dump", connections=[]) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with([], []) -async def test_message_to_notify_dump_one( +async def test_handle_message_dump_one( + subject: helpers.ModuleListener, mock_callback: AsyncMock, connections: List[models.Connection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to load.""" message = models.Message(status="dump", connections=connections[:1]) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with(modules_at_port[:1], []) -async def test_message_to_notify_dump_many( +async def test_handle_message_dump_many( + subject: helpers.ModuleListener, mock_callback: AsyncMock, connections: List[models.Connection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to load.""" message = models.Message(status="dump", connections=connections) - await helpers.ModuleListener.message_to_notify( - message=message, notify_method=mock_callback + await subject.handle_message( + message=message ) mock_callback.assert_called_once_with(modules_at_port, []) diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index 961db324d79..63ef0203b6f 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -62,14 +62,14 @@ def _get_loop() -> asyncio.AbstractEventLoop: @staticmethod def _start_emulation_app(emulator_settings: Settings) -> Process: """Start emulated OT-2""" - modules = [ModuleType.Thermocycler, ModuleType.Magnetic, ModuleType.Temperature] + modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] - def runit(): + def _run_app(): asyncio.run( run_app.run(emulator_settings, modules=[m.value for m in modules]) ) - proc = Process(target=runit) + proc = Process(target=_run_app) proc.daemon = True proc.start() @@ -80,9 +80,13 @@ async def _wait_ready() -> None: await wait_emulators(client=c, modules=modules, timeout=5) c.close() - proc2 = Process(target=lambda: asyncio.run(_wait_ready())) - proc2.start() - proc2.join() + def _run_wait_ready(): + asyncio.run(_wait_ready()) + + ready_proc = Process(target=_run_wait_ready) + ready_proc.daemon = True + ready_proc.start() + ready_proc.join() return proc From 085c030186bcc1084e9edcbdd340bfa882f992ed Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 27 Oct 2021 11:04:02 -0400 Subject: [PATCH 11/19] expand module emulation settings. --- .../hardware_control/emulation/magdeck.py | 17 +++--- .../emulation/module_server/helpers.py | 2 +- .../emulation/scripts/run_module_emulator.py | 8 ++- .../hardware_control/emulation/settings.py | 43 +++++++++++++- .../hardware_control/emulation/tempdeck.py | 24 +++++--- .../emulation/thermocycler.py | 25 +++++---- .../emulation/module_server/test_helpers.py | 56 ++++++++----------- .../hardware_control/integration/conftest.py | 45 +++++++++------ 8 files changed, 138 insertions(+), 82 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/magdeck.py b/api/src/opentrons/hardware_control/emulation/magdeck.py index bfcc8d39587..eaa7eb718d1 100644 --- a/api/src/opentrons/hardware_control/emulation/magdeck.py +++ b/api/src/opentrons/hardware_control/emulation/magdeck.py @@ -8,24 +8,21 @@ from opentrons.drivers.mag_deck.driver import GCODE from opentrons.hardware_control.emulation.parser import Parser, Command from .abstract_emulator import AbstractEmulator +from .settings import MagDeckSettings logger = logging.getLogger(__name__) -SERIAL = "magnetic_emulator" -MODEL = "mag_deck_v20" -VERSION = "2.0.0" - - class MagDeckEmulator(AbstractEmulator): """Magdeck emulator""" height: float = 0 position: float = 0 - def __init__(self, parser: Parser) -> None: - self.reset() + def __init__(self, parser: Parser, settings: MagDeckSettings) -> None: + self._settings = settings self._parser = parser + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -53,7 +50,11 @@ def _handle(self, command: Command) -> Optional[str]: elif command.gcode == GCODE.GET_CURRENT_POSITION: return f"Z:{self.position}" elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) elif command.gcode == GCODE.PROGRAMMING_MODE: pass return None diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index 4f92f8d79be..4d17072d866 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -60,6 +60,7 @@ async def handle_message(self, message: Message) -> None: Returns: None """ + def _next_index() -> int: index = self._hub_index self._hub_index += 1 @@ -79,7 +80,6 @@ def _next_index() -> int: await self._notify_method([], connections) - async def wait_emulators( client: ModuleServerClient, modules: Sequence[ModuleType], diff --git a/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py b/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py index 4c19fe539d2..e05ea632c98 100644 --- a/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py @@ -17,9 +17,11 @@ from opentrons.hardware_control.emulation.settings import Settings, ProxySettings emulator_builder: Final[Dict[str, Callable[[Settings], AbstractEmulator]]] = { - ModuleType.Magnetic.value: lambda s: MagDeckEmulator(Parser()), - ModuleType.Temperature.value: lambda s: TempDeckEmulator(Parser()), - ModuleType.Thermocycler.value: lambda s: ThermocyclerEmulator(Parser()), + ModuleType.Magnetic.value: lambda s: MagDeckEmulator(Parser(), s.magdeck), + ModuleType.Temperature.value: lambda s: TempDeckEmulator(Parser(), s.tempdeck), + ModuleType.Thermocycler.value: lambda s: ThermocyclerEmulator( + Parser(), s.thermocycler + ), } emulator_port: Final[Dict[str, Callable[[Settings], ProxySettings]]] = { diff --git a/api/src/opentrons/hardware_control/emulation/settings.py b/api/src/opentrons/hardware_control/emulation/settings.py index 192847393a4..db5e1cbe860 100644 --- a/api/src/opentrons/hardware_control/emulation/settings.py +++ b/api/src/opentrons/hardware_control/emulation/settings.py @@ -1,3 +1,4 @@ +from opentrons.hardware_control.emulation.util import TEMPERATURE_ROOM from pydantic import BaseSettings, BaseModel @@ -17,7 +18,31 @@ class SmoothieSettings(BaseModel): port: int = 9996 -class ProxySettings(BaseSettings): +class BaseModuleSettings(BaseModel): + serial_number: str + model: str + version: str + + +class TemperatureModelSettings(BaseModel): + degrees_per_tick: float = 2.0 + starting: float = float(TEMPERATURE_ROOM) + + +class MagDeckSettings(BaseModuleSettings): + pass + + +class TempDeckSettings(BaseModuleSettings): + temperature: TemperatureModelSettings + + +class ThermocyclerSettings(BaseModuleSettings): + lid_temperature: TemperatureModelSettings + plate_temperature: TemperatureModelSettings + + +class ProxySettings(BaseModel): """Settings for a proxy.""" host: str = "0.0.0.0" @@ -34,6 +59,22 @@ class ModuleServerSettings(BaseModel): class Settings(BaseSettings): smoothie: SmoothieSettings = SmoothieSettings() + magdeck: MagDeckSettings = MagDeckSettings( + serial_number="magnetic_emulator", model="mag_deck_v20", version="2.0.0" + ) + tempdeck: TempDeckSettings = TempDeckSettings( + serial_number="temperature_emulator", + model="temp_deck_v20", + version="v2.0.1", + temperature=TemperatureModelSettings(starting=0.0), + ) + thermocycler: ThermocyclerSettings = ThermocyclerSettings( + serial_number="thermocycler_emulator", + model="v02", + version="v1.1.0", + lid_temperature=TemperatureModelSettings(), + plate_temperature=TemperatureModelSettings(), + ) host: str = "0.0.0.0" diff --git a/api/src/opentrons/hardware_control/emulation/tempdeck.py b/api/src/opentrons/hardware_control/emulation/tempdeck.py index 9e99c1e23a8..22c237ee032 100644 --- a/api/src/opentrons/hardware_control/emulation/tempdeck.py +++ b/api/src/opentrons/hardware_control/emulation/tempdeck.py @@ -9,6 +9,7 @@ from opentrons.drivers.temp_deck.driver import GCODE from opentrons.hardware_control.emulation import util from opentrons.hardware_control.emulation.parser import Parser, Command +from opentrons.hardware_control.emulation.settings import TempDeckSettings from .abstract_emulator import AbstractEmulator from .simulations import Temperature @@ -17,17 +18,15 @@ logger = logging.getLogger(__name__) -SERIAL = "temperature_emulator" -MODEL = "temp_deck_v20" -VERSION = "v2.0.1" - - class TempDeckEmulator(AbstractEmulator): """TempDeck emulator""" - def __init__(self, parser: Parser) -> None: - self.reset() + _temperature: Temperature + + def __init__(self, parser: Parser, settings: TempDeckSettings) -> None: + self._settings = settings self._parser = parser + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -36,7 +35,10 @@ def handle(self, line: str) -> Optional[str]: return None if not joined else joined def reset(self): - self._temperature = Temperature(per_tick=0.25, current=0.0) + self._temperature = Temperature( + per_tick=self._settings.temperature.degrees_per_tick, + current=self._settings.temperature.starting, + ) def _handle(self, command: Command) -> Optional[str]: """Handle a command.""" @@ -57,7 +59,11 @@ def _handle(self, command: Command) -> Optional[str]: elif command.gcode == GCODE.DISENGAGE: self._temperature.deactivate(util.TEMPERATURE_ROOM) elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) elif command.gcode == GCODE.PROGRAMMING_MODE: pass return None diff --git a/api/src/opentrons/hardware_control/emulation/thermocycler.py b/api/src/opentrons/hardware_control/emulation/thermocycler.py index f841b00addd..e35f0d05214 100644 --- a/api/src/opentrons/hardware_control/emulation/thermocycler.py +++ b/api/src/opentrons/hardware_control/emulation/thermocycler.py @@ -8,6 +8,7 @@ from opentrons.drivers.thermocycler.driver import GCODE from opentrons.drivers.types import ThermocyclerLidStatus from opentrons.hardware_control.emulation.parser import Parser, Command +from opentrons.hardware_control.emulation.settings import ThermocyclerSettings from .abstract_emulator import AbstractEmulator from .simulations import Temperature, TemperatureWithHold @@ -16,11 +17,6 @@ logger = logging.getLogger(__name__) -SERIAL = "thermocycler_emulator" -MODEL = "v02" -VERSION = "v1.1.0" - - class ThermocyclerEmulator(AbstractEmulator): """Thermocycler emulator""" @@ -30,9 +26,10 @@ class ThermocyclerEmulator(AbstractEmulator): plate_volume: util.OptionalValue[float] plate_ramp_rate: util.OptionalValue[float] - def __init__(self, parser: Parser) -> None: - self.reset() + def __init__(self, parser: Parser, settings: ThermocyclerSettings) -> None: self._parser = parser + self._settings = settings + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -41,9 +38,13 @@ def handle(self, line: str) -> Optional[str]: return None if not joined else joined def reset(self): - self._lid_temperature = Temperature(per_tick=2, current=util.TEMPERATURE_ROOM) + self._lid_temperature = Temperature( + per_tick=self._settings.lid_temperature.degrees_per_tick, + current=self._settings.lid_temperature.starting, + ) self._plate_temperature = TemperatureWithHold( - per_tick=2, current=util.TEMPERATURE_ROOM + per_tick=self._settings.plate_temperature.degrees_per_tick, + current=self._settings.plate_temperature.starting, ) self.lid_status = ThermocyclerLidStatus.OPEN self.plate_volume = util.OptionalValue[float]() @@ -115,7 +116,11 @@ def _handle(self, command: Command) -> Optional[str]: # noqa: C901 elif command.gcode == GCODE.DEACTIVATE_BLOCK: self._plate_temperature.deactivate(temperature=util.TEMPERATURE_ROOM) elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) return None @staticmethod diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py index d6a00d94207..a8852860db6 100644 --- a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py +++ b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py @@ -3,8 +3,10 @@ import pytest from mock import AsyncMock from opentrons.drivers.rpi_drivers.types import USBPort -from opentrons.hardware_control.emulation.module_server import helpers, \ - ModuleServerClient +from opentrons.hardware_control.emulation.module_server import ( + helpers, + ModuleServerClient, +) from opentrons.hardware_control.emulation.module_server import models from opentrons.hardware_control.modules import ModuleAtPort @@ -45,18 +47,18 @@ def modules_at_port() -> List[ModuleAtPort]: ModuleAtPort( port=f"url{i}", name=f"module_type{i}", - usb_port=USBPort(name=f"identifier{i}", sub_names=[], hub=i), + usb_port=USBPort(name=f"identifier{i}", sub_names=[], hub=i + 1), ) for i in range(5) ] -async def test_handle_message_connected_empty(subject: helpers.ModuleListener, mock_callback: AsyncMock) -> None: +async def test_handle_message_connected_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: """It should call the call back with the correct modules to add.""" message = models.Message(status="connected", connections=[]) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with([], []) @@ -68,9 +70,7 @@ async def test_handle_message_connected_one( ) -> None: """It should call the call back with the correct modules to add.""" message = models.Message(status="connected", connections=connections[:1]) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with(modules_at_port[:1], []) @@ -82,18 +82,16 @@ async def test_handle_message_connected_many( ) -> None: """It should call the call back with the correct modules to add.""" message = models.Message(status="connected", connections=connections) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with(modules_at_port, []) -async def test_handle_message_disconnected_empty(subject: helpers.ModuleListener, mock_callback: AsyncMock) -> None: +async def test_handle_message_disconnected_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: """It should call the call back with the correct modules to remove.""" message = models.Message(status="disconnected", connections=[]) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with([], []) @@ -105,9 +103,7 @@ async def test_handle_message_disconnected_one( ) -> None: """It should call the call back with the correct modules to remove.""" message = models.Message(status="disconnected", connections=connections[:1]) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with([], modules_at_port[:1]) @@ -119,18 +115,16 @@ async def test_handle_message_disconnected_many( ) -> None: """It should call the call back with the correct modules to remove.""" message = models.Message(status="disconnected", connections=connections) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with([], modules_at_port) -async def test_handle_message_dump_empty(subject: helpers.ModuleListener, mock_callback: AsyncMock) -> None: +async def test_handle_message_dump_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: """It should call the call back with the correct modules to load.""" message = models.Message(status="dump", connections=[]) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with([], []) @@ -142,9 +136,7 @@ async def test_handle_message_dump_one( ) -> None: """It should call the call back with the correct modules to load.""" message = models.Message(status="dump", connections=connections[:1]) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with(modules_at_port[:1], []) @@ -156,7 +148,5 @@ async def test_handle_message_dump_many( ) -> None: """It should call the call back with the correct modules to load.""" message = models.Message(status="dump", connections=connections) - await subject.handle_message( - message=message - ) + await subject.handle_message(message=message) mock_callback.assert_called_once_with(modules_at_port, []) diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 3ac09b54640..b184a621a19 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -1,8 +1,12 @@ +from multiprocessing import Process from typing import Iterator import pytest import threading import asyncio + +from opentrons.hardware_control.emulation.module_server import ModuleServerClient +from opentrons.hardware_control.emulation.module_server.helpers import wait_emulators from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control.emulation.types import ModuleType from opentrons.hardware_control.emulation.settings import ( @@ -27,24 +31,31 @@ def emulator_settings() -> Settings: @pytest.fixture(scope="session") def emulation_app(emulator_settings: Settings) -> Iterator[None]: """Run the emulators""" + modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] + + def _run_app() -> None: + asyncio.run(run_app.run(emulator_settings, modules=[m.value for m in modules])) + + proc = Process(target=_run_app) + proc.daemon = True + proc.start() - def runit() -> None: - asyncio.run( - run_app.run( - settings=emulator_settings, - modules=[ - ModuleType.Magnetic, - ModuleType.Temperature, - ModuleType.Thermocycler, - ], - ) + async def _wait_ready() -> None: + c = await ModuleServerClient.connect( + host="localhost", port=emulator_settings.module_server.port ) + await wait_emulators(client=c, modules=modules, timeout=5) + c.close() + + def _run_wait_ready() -> None: + asyncio.run(_wait_ready()) + + ready_proc = Process(target=_run_wait_ready) + ready_proc.daemon = True + ready_proc.start() + ready_proc.join() - # TODO 20210219 - # The emulators must be run in a separate thread because our serial - # drivers block the main thread. Remove this thread when that is no - # longer true. - t = threading.Thread(target=runit) - t.daemon = True - t.start() yield + + proc.kill() + proc.join() From 5903b8cee28fae75c10ff978f2201ec11daf3d99 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 27 Oct 2021 12:18:45 -0400 Subject: [PATCH 12/19] update docker readme. --- DOCKER.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/DOCKER.md b/DOCKER.md index 6a02c7f459e..d1483b379a1 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -37,6 +37,61 @@ For example to use a `p300_multi` on the right add: OT_EMULATOR_smoothie: '{"right": {"model": "p300_multi"}}' ``` + +### Adding more emulators + +#### Magdeck + +To add a second mag deck emulator make a copy of the existing `magdeck` section and change the key and `serial_number`. + +For example this adds a `magdeck` with the serial number `magdeck2`: +``` + magdeck2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator magdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_magdeck: '{"serial_number": "magdeck2", "model":"mag_deck_v20", "version":"2.0.0"}' +``` + +#### Tempdeck + +To add a second temp deck emulator make a copy of the existing `tempdeck` section and change the key and `serial_number`. + +For example this adds a `tempdeck` with the serial number `tempdeck2`: +``` + tempdeck2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator tempdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_tempdeck: '{"serial_number": "tempdeck2", "model":"temp_deck_v20", "version":"v2.0.1", "temperature": {"starting":0.0, "degrees_per_tick": 2.0}}' +``` + +#### Thermocycler + +To add a second thermocycler emulator make a copy of the existing `thermocycler` section and change the key and `serial_number`. + +For example this adds a `thermocycler` with the serial number `thermocycler2`: +``` + thermocycler2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator thermocycler emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_thermocycler: '{"serial_number": "thermocycler2", "model":"v02", "version":"v1.1.0", "lid_temperature": {"starting":23.0, "degrees_per_tick": 2.0}, "plate_temperature": {"starting":23.0, "degrees_per_tick": 2.0}}' +``` + + ## Known Issues - Pipettes cannot be changed at run time. From 139179fdcc1c06cf16586220ef2243a0cf54209c Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 27 Oct 2021 12:54:07 -0400 Subject: [PATCH 13/19] format-js --- DOCKER.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index d1483b379a1..3bb4026b34f 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -37,14 +37,14 @@ For example to use a `p300_multi` on the right add: OT_EMULATOR_smoothie: '{"right": {"model": "p300_multi"}}' ``` - ### Adding more emulators #### Magdeck -To add a second mag deck emulator make a copy of the existing `magdeck` section and change the key and `serial_number`. +To add a second mag deck emulator make a copy of the existing `magdeck` section and change the key and `serial_number`. For example this adds a `magdeck` with the serial number `magdeck2`: + ``` magdeck2: build: . @@ -59,9 +59,10 @@ For example this adds a `magdeck` with the serial number `magdeck2`: #### Tempdeck -To add a second temp deck emulator make a copy of the existing `tempdeck` section and change the key and `serial_number`. +To add a second temp deck emulator make a copy of the existing `tempdeck` section and change the key and `serial_number`. For example this adds a `tempdeck` with the serial number `tempdeck2`: + ``` tempdeck2: build: . @@ -76,9 +77,10 @@ For example this adds a `tempdeck` with the serial number `tempdeck2`: #### Thermocycler -To add a second thermocycler emulator make a copy of the existing `thermocycler` section and change the key and `serial_number`. +To add a second thermocycler emulator make a copy of the existing `thermocycler` section and change the key and `serial_number`. For example this adds a `thermocycler` with the serial number `thermocycler2`: + ``` thermocycler2: build: . @@ -91,7 +93,6 @@ For example this adds a `thermocycler` with the serial number `thermocycler2`: OT_EMULATOR_thermocycler: '{"serial_number": "thermocycler2", "model":"v02", "version":"v1.1.0", "lid_temperature": {"starting":23.0, "degrees_per_tick": 2.0}, "plate_temperature": {"starting":23.0, "degrees_per_tick": 2.0}}' ``` - ## Known Issues - Pipettes cannot be changed at run time. From 42288680478316e71eda9fd680ad2d561865994a Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 27 Oct 2021 13:09:12 -0400 Subject: [PATCH 14/19] lint --- api/src/opentrons/hardware_control/module_control.py | 3 --- api/tests/opentrons/hardware_control/integration/conftest.py | 1 - 2 files changed, 4 deletions(-) diff --git a/api/src/opentrons/hardware_control/module_control.py b/api/src/opentrons/hardware_control/module_control.py index 46291779d05..f7a5e43eb5e 100644 --- a/api/src/opentrons/hardware_control/module_control.py +++ b/api/src/opentrons/hardware_control/module_control.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging import asyncio -import os import re from typing import List, Tuple, Optional from glob import glob @@ -11,8 +10,6 @@ from opentrons.hardware_control.emulation.module_server.helpers import ( listen_module_connection, ) -from opentrons.hardware_control.modules import ModuleAtPort - from .types import AionotifyEvent, BoardRevision from . import modules diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index b184a621a19..fa8dc63774f 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -2,7 +2,6 @@ from typing import Iterator import pytest -import threading import asyncio from opentrons.hardware_control.emulation.module_server import ModuleServerClient From c05f50007b3824858032d32d7591527d37b911ee Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 27 Oct 2021 13:41:08 -0400 Subject: [PATCH 15/19] go back to threadings. I don't want to debug windows nonsense. --- .../hardware_control/integration/conftest.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index fa8dc63774f..d96bd4e04e7 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -1,5 +1,5 @@ -from multiprocessing import Process from typing import Iterator +import threading import pytest import asyncio @@ -35,10 +35,6 @@ def emulation_app(emulator_settings: Settings) -> Iterator[None]: def _run_app() -> None: asyncio.run(run_app.run(emulator_settings, modules=[m.value for m in modules])) - proc = Process(target=_run_app) - proc.daemon = True - proc.start() - async def _wait_ready() -> None: c = await ModuleServerClient.connect( host="localhost", port=emulator_settings.module_server.port @@ -49,12 +45,15 @@ async def _wait_ready() -> None: def _run_wait_ready() -> None: asyncio.run(_wait_ready()) - ready_proc = Process(target=_run_wait_ready) + # Start the emulator thread. + t = threading.Thread(target=_run_app) + t.daemon = True + t.start() + + # Start the wait for emulator ready thread and wait for it to terminate. + ready_proc = threading.Thread(target=_run_wait_ready) ready_proc.daemon = True ready_proc.start() ready_proc.join() yield - - proc.kill() - proc.join() From c7553cb0fbe99d1ff6746f6b74fb763758a4917c Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 27 Oct 2021 16:12:18 -0400 Subject: [PATCH 16/19] fix bug. --- .../g_code_test_data/g_code_configuration.py | 4 ++-- .../g_code_test_data/http/modules/magdeck.py | 8 ++++---- .../g_code_test_data/http/modules/tempdeck.py | 8 ++++---- .../http/modules/thermocycler.py | 17 ++++++++--------- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/g-code-testing/g_code_test_data/g_code_configuration.py b/g-code-testing/g_code_test_data/g_code_configuration.py index a26f8a78a54..b80edcf092b 100644 --- a/g-code-testing/g_code_test_data/g_code_configuration.py +++ b/g-code-testing/g_code_test_data/g_code_configuration.py @@ -84,7 +84,7 @@ class ProtocolGCodeConfirmConfig(BaseModel, SharedFunctionsMixin): name: constr(regex=r'^[a-z0-9_]*$') path: str settings: Settings - driver = 'protocol' + driver: str = 'protocol' marks: List[Mark] = [pytest.mark.g_code_confirm] class Config: @@ -99,7 +99,7 @@ class HTTPGCodeConfirmConfig(BaseModel, SharedFunctionsMixin): name: constr(regex=r'^[a-z0-9_]*$') executable: Callable settings: Settings - driver = 'http' + driver: str = 'http' marks: List[Mark] = [pytest.mark.g_code_confirm] class Config: diff --git a/g-code-testing/g_code_test_data/http/modules/magdeck.py b/g-code-testing/g_code_test_data/http/modules/magdeck.py index 79054e5faf9..e1dc2b573de 100644 --- a/g-code-testing/g_code_test_data/http/modules/magdeck.py +++ b/g-code-testing/g_code_test_data/http/modules/magdeck.py @@ -3,14 +3,14 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.magdeck import SERIAL as SERIAL_NUM + MAGDECK_CALIBRATE = HTTPGCodeConfirmConfig( name='magdeck_calibrate', executable=partial( post_serial_command, command=SerialCommand(command_type='calibrate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -20,7 +20,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -30,7 +30,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='engage', args=[5.1]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) diff --git a/g-code-testing/g_code_test_data/http/modules/tempdeck.py b/g-code-testing/g_code_test_data/http/modules/tempdeck.py index 6b13fce9e43..6ff1a640838 100644 --- a/g-code-testing/g_code_test_data/http/modules/tempdeck.py +++ b/g-code-testing/g_code_test_data/http/modules/tempdeck.py @@ -3,14 +3,14 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.tempdeck import SERIAL as SERIAL_NUM + TEMPDECK_DEACTIVATE = HTTPGCodeConfirmConfig( name='tempdeck_deactivate', executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -22,7 +22,7 @@ # Keep the args at a low value because the temp starts and 0.0 and only # changes 0.25 degrees every second command=SerialCommand(command_type='set_temperature', args=[2.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -34,7 +34,7 @@ # This function does not wait on the tempdeck to finish coming to temp # so no need to set to a low value command=SerialCommand(command_type='start_set_temperature', args=[40.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) diff --git a/g-code-testing/g_code_test_data/http/modules/thermocycler.py b/g-code-testing/g_code_test_data/http/modules/thermocycler.py index 37ec7ba13bb..1485f8719c5 100644 --- a/g-code-testing/g_code_test_data/http/modules/thermocycler.py +++ b/g-code-testing/g_code_test_data/http/modules/thermocycler.py @@ -3,7 +3,6 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.thermocycler import SERIAL as SERIAL_NUM THERMOCYCLER_CLOSE = HTTPGCodeConfirmConfig( @@ -11,7 +10,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='close'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -21,7 +20,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='open'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -31,7 +30,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -41,7 +40,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate_block'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -51,7 +50,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate_lid'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -66,7 +65,7 @@ , 1 ] ), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -76,7 +75,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='set_lid_temperature', args=[37.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -86,7 +85,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='set_temperature', args=[1.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) From 6b11e53c93e53b9bc7d8693da102d64545d85b61 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Fri, 29 Oct 2021 09:27:38 -0400 Subject: [PATCH 17/19] redo gcode testing's emulator setup. --- .../emulation/module_server/client.py | 14 +++- .../emulation/module_server/helpers.py | 13 +++- .../hardware_control/emulation/settings.py | 2 - .../hardware_control/integration/conftest.py | 5 +- g-code-testing/cli.py | 2 +- .../g_code_parsing/g_code_engine.py | 72 +++++++++---------- .../g_code_parsing/test_g_code_engine.py | 1 - 7 files changed, 60 insertions(+), 49 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/client.py b/api/src/opentrons/hardware_control/emulation/module_server/client.py index f0f6652e6fa..4749ef68f10 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/client.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/client.py @@ -1,11 +1,16 @@ from __future__ import annotations import asyncio +from asyncio import IncompleteReadError, LimitOverrunError from typing import Optional from opentrons.hardware_control.emulation.module_server.models import Message from opentrons.hardware_control.emulation.module_server.server import MessageDelimiter +class ModuleServerClientError(Exception): + pass + + class ModuleServerClient: """A module server client.""" @@ -55,9 +60,12 @@ async def connect( async def read(self) -> Message: """Read a message from the module server.""" - b = await self._reader.readuntil(MessageDelimiter) - m: Message = Message.parse_raw(b) - return m + try: + b = await self._reader.readuntil(MessageDelimiter) + m: Message = Message.parse_raw(b) + return m + except (IncompleteReadError, LimitOverrunError) as e: + raise ModuleServerClientError(str(e)) def close(self) -> None: """Close the client.""" diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index 4d17072d866..cd075a91727 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -2,7 +2,10 @@ from typing import Sequence, Set, Callable, List, Awaitable from opentrons.drivers.rpi_drivers.types import USBPort -from opentrons.hardware_control.emulation.module_server.client import ModuleServerClient +from opentrons.hardware_control.emulation.module_server.client import ( + ModuleServerClient, + ModuleServerClientError, +) from opentrons.hardware_control.emulation.module_server.models import Message from opentrons.hardware_control.emulation.module_server.server import log from opentrons.hardware_control.emulation.settings import Settings @@ -48,8 +51,12 @@ def __init__(self, client: ModuleServerClient, notify_method: NotifyMethod) -> N async def run(self) -> None: """Run the listener.""" while True: - m = await self._client.read() - await self.handle_message(message=m) + try: + m = await self._client.read() + await self.handle_message(message=m) + except ModuleServerClientError: + log.exception("Read error.") + break async def handle_message(self, message: Message) -> None: """Call callback with results of message. diff --git a/api/src/opentrons/hardware_control/emulation/settings.py b/api/src/opentrons/hardware_control/emulation/settings.py index db5e1cbe860..d81de05275f 100644 --- a/api/src/opentrons/hardware_control/emulation/settings.py +++ b/api/src/opentrons/hardware_control/emulation/settings.py @@ -76,8 +76,6 @@ class Settings(BaseSettings): plate_temperature=TemperatureModelSettings(), ) - host: str = "0.0.0.0" - heatershaker_proxy: ProxySettings = ProxySettings( emulator_port=9000, driver_port=9995 ) diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index d96bd4e04e7..6590766fa34 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -19,7 +19,6 @@ def emulator_settings() -> Settings: """Emulator settings""" return Settings( - host="0.0.0.0", smoothie=SmoothieSettings( left=PipetteSettings(model="p20_multi_v2.0", id="P3HMV202020041605"), right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), @@ -37,7 +36,9 @@ def _run_app() -> None: async def _wait_ready() -> None: c = await ModuleServerClient.connect( - host="localhost", port=emulator_settings.module_server.port + host="localhost", + port=emulator_settings.module_server.port, + interval_seconds=1, ) await wait_emulators(client=c, modules=modules, timeout=5) c.close() diff --git a/g-code-testing/cli.py b/g-code-testing/cli.py index d0481f3b346..8cb73ae1c57 100644 --- a/g-code-testing/cli.py +++ b/g-code-testing/cli.py @@ -78,7 +78,7 @@ def _diff(self) -> str: else: text = "No difference between compared strings" - return text + return text def _configurations(self) -> str: """Get a list of runnable G-Code Configurations""" diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index 63ef0203b6f..06aed89b099 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -1,8 +1,9 @@ import os import sys import asyncio +import time from multiprocessing import Process -from typing import Generator, Callable +from typing import Generator, Callable, Iterator from collections import namedtuple from opentrons.hardware_control.emulation.settings import Settings @@ -59,23 +60,24 @@ def _get_loop() -> asyncio.AbstractEventLoop: asyncio.set_event_loop(_loop) return asyncio.get_event_loop() - @staticmethod - def _start_emulation_app(emulator_settings: Settings) -> Process: - """Start emulated OT-2""" + @contextmanager + def _emulate(self) -> Iterator[ThreadManager]: + """Context manager that starts emulated OT-2 hardware environment. A + hardware controller is returned.""" modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] + # Entry point for the emulator app process def _run_app(): - asyncio.run( - run_app.run(emulator_settings, modules=[m.value for m in modules]) - ) + asyncio.run(run_app.run(self._config, modules=[m.value for m in modules])) proc = Process(target=_run_app) proc.daemon = True proc.start() + # Entry point for process that waits for emulation to be ready. async def _wait_ready() -> None: c = await ModuleServerClient.connect( - host="localhost", port=emulator_settings.module_server.port + host="localhost", port=self._config.module_server.port ) await wait_emulators(client=c, modules=modules, timeout=5) c.close() @@ -88,18 +90,22 @@ def _run_wait_ready(): ready_proc.start() ready_proc.join() - return proc - - @staticmethod - def _emulate_hardware(settings: Settings) -> ThreadManager: - """Created emulated smoothie""" + # Hardware controller conf = build_config({}) emulator = ThreadManager( API.build_hardware_controller, conf, - GCodeEngine.URI_TEMPLATE % settings.smoothie.port, + GCodeEngine.URI_TEMPLATE % self._config.smoothie.port, ) - return emulator + # Wait for modules to be present + while len(emulator.attached_modules) != len(modules): + time.sleep(0.1) + + yield emulator + + # Finished. Stop the emulator + proc.kill() + proc.join() @staticmethod def _get_protocol(file_path: str) -> Protocol: @@ -118,20 +124,16 @@ def run_protocol(self, path: str) -> Generator: :return: GCodeProgram with all the parsed data """ file_path = os.path.join(get_configuration_dir(), path) - app_process = self._start_emulation_app(emulator_settings=self._config) - protocol = self._get_protocol(file_path) - context = ProtocolContext( - implementation=ProtocolContextImplementation( - hardware=self._emulate_hardware(settings=self._config) - ), - loop=self._get_loop(), - ) - parsed_protocol = parse(protocol.text, protocol.filename) - with GCodeWatcher(emulator_settings=self._config) as watcher: - execute.run_protocol(parsed_protocol, context=context) - yield GCodeProgram.from_g_code_watcher(watcher) - app_process.terminate() - app_process.join() + with self._emulate() as h: + protocol = self._get_protocol(file_path) + context = ProtocolContext( + implementation=ProtocolContextImplementation(hardware=h), + loop=self._get_loop(), + ) + parsed_protocol = parse(protocol.text, protocol.filename) + with GCodeWatcher(emulator_settings=self._config) as watcher: + execute.run_protocol(parsed_protocol, context=context) + yield GCodeProgram.from_g_code_watcher(watcher) @contextmanager def run_http(self, executable: Callable): @@ -140,11 +142,7 @@ def run_http(self, executable: Callable): :param executable: Function connected to HTTP Request to execute :return: """ - app_process = self._start_emulation_app(emulator_settings=self._config) - with GCodeWatcher(emulator_settings=self._config) as watcher: - asyncio.run( - executable(hardware=self._emulate_hardware(settings=self._config)) - ) - yield GCodeProgram.from_g_code_watcher(watcher) - app_process.terminate() - app_process.join() + with self._emulate() as h: + with GCodeWatcher(emulator_settings=self._config) as watcher: + asyncio.run(executable(hardware=h)) + yield GCodeProgram.from_g_code_watcher(watcher) diff --git a/g-code-testing/tests/g_code_parsing/test_g_code_engine.py b/g-code-testing/tests/g_code_parsing/test_g_code_engine.py index 1a5556628bc..e85f6b423c7 100644 --- a/g-code-testing/tests/g_code_parsing/test_g_code_engine.py +++ b/g-code-testing/tests/g_code_parsing/test_g_code_engine.py @@ -13,7 +13,6 @@ from g_code_parsing.utils import get_configuration_dir CONFIG = Settings( - host="0.0.0.0", smoothie=SmoothieSettings( left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), From ea431f2b87c25810b9502bed729f295c19dadfb1 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Fri, 29 Oct 2021 10:56:32 -0400 Subject: [PATCH 18/19] documentation. --- .../emulation/module_server/__init__.py | 6 ++--- .../emulation/module_server/client.py | 6 ++--- .../emulation/module_server/helpers.py | 12 +++++---- .../emulation/module_server/models.py | 25 +++++++++++++------ .../emulation/module_server/server.py | 14 ++++++++--- .../hardware_control/emulation/proxy.py | 6 ++++- .../emulation/module_server/test_helpers.py | 20 +++++++-------- .../hardware_control/integration/conftest.py | 4 +-- 8 files changed, 58 insertions(+), 35 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/__init__.py b/api/src/opentrons/hardware_control/emulation/module_server/__init__.py index fd9f7229550..082295dbfc7 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/__init__.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/__init__.py @@ -1,8 +1,8 @@ +"""Package for the module status server.""" from .server import ModuleStatusServer -from .client import ModuleServerClient - +from .client import ModuleStatusClient __all__ = [ "ModuleStatusServer", - "ModuleServerClient", + "ModuleStatusClient", ] diff --git a/api/src/opentrons/hardware_control/emulation/module_server/client.py b/api/src/opentrons/hardware_control/emulation/module_server/client.py index 4749ef68f10..b1051861fc6 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/client.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/client.py @@ -11,7 +11,7 @@ class ModuleServerClientError(Exception): pass -class ModuleServerClient: +class ModuleStatusClient: """A module server client.""" def __init__( @@ -28,7 +28,7 @@ async def connect( port: int, retries: int = 3, interval_seconds: float = 0.1, - ) -> ModuleServerClient: + ) -> ModuleStatusClient: """Connect to the module server. Args: @@ -52,7 +52,7 @@ async def connect( await asyncio.sleep(interval_seconds) if r is not None and w is not None: - return ModuleServerClient(reader=r, writer=w) + return ModuleStatusClient(reader=r, writer=w) else: raise IOError( f"Failed to connect to module_server at after {retries} retries." diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py index cd075a91727..ad4e7ffbabe 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -1,9 +1,11 @@ +"""Utililty methods and classes for interacting with the Module Status Server.""" + import asyncio from typing import Sequence, Set, Callable, List, Awaitable from opentrons.drivers.rpi_drivers.types import USBPort from opentrons.hardware_control.emulation.module_server.client import ( - ModuleServerClient, + ModuleStatusClient, ModuleServerClientError, ) from opentrons.hardware_control.emulation.module_server.models import Message @@ -20,7 +22,7 @@ async def listen_module_connection(callback: NotifyMethod) -> None: """Listen for module emulator connections.""" settings = Settings() try: - client = await ModuleServerClient.connect( + client = await ModuleStatusClient.connect( host=settings.module_server.host, port=settings.module_server.port, interval_seconds=1.0, @@ -34,7 +36,7 @@ async def listen_module_connection(callback: NotifyMethod) -> None: class ModuleListener: """Provide a callback for listening for new and removed module connections.""" - def __init__(self, client: ModuleServerClient, notify_method: NotifyMethod) -> None: + def __init__(self, client: ModuleStatusClient, notify_method: NotifyMethod) -> None: """Constructor. Args: @@ -88,7 +90,7 @@ def _next_index() -> int: async def wait_emulators( - client: ModuleServerClient, + client: ModuleStatusClient, modules: Sequence[ModuleType], timeout: float, ) -> None: @@ -105,7 +107,7 @@ async def wait_emulators( asyncio.TimeoutError on timeout. """ - async def _wait_modules(cl: ModuleServerClient, module_set: Set[str]) -> None: + async def _wait_modules(cl: ModuleStatusClient, module_set: Set[str]) -> None: """Read messages from module server waiting for modules in module_set to be connected.""" while module_set: diff --git a/api/src/opentrons/hardware_control/emulation/module_server/models.py b/api/src/opentrons/hardware_control/emulation/module_server/models.py index f2c4b0c598e..05363bfc109 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/models.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/models.py @@ -2,19 +2,30 @@ from typing_extensions import Literal -from pydantic import BaseModel +from pydantic import BaseModel, Field -class Connection(BaseModel): +class ModuleConnection(BaseModel): """Model a single module connection.""" - url: str - module_type: str - identifier: str + url: str = Field( + ..., + description="The url (port) value the module driver should connect to. " + "For example: socket://host:port", + ) + module_type: str = Field( + ..., description="What kind of module this connection emulates." + ) + identifier: str = Field(..., description="Unique id for this emulator.") class Message(BaseModel): """A message sent to module server clients.""" - status: Literal["connected", "disconnected", "dump"] - connections: List[Connection] + status: Literal["connected", "disconnected", "dump"] = Field( + ..., + description="`dump` includes a complete list of connected emulators. " + "`connected` for new connections. `disconnected` for emulators " + "that have disconnected. ", + ) + connections: List[ModuleConnection] diff --git a/api/src/opentrons/hardware_control/emulation/module_server/server.py b/api/src/opentrons/hardware_control/emulation/module_server/server.py index 13da6410392..5a3d696eb7b 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server/server.py +++ b/api/src/opentrons/hardware_control/emulation/module_server/server.py @@ -1,9 +1,10 @@ +"""Server notifying of module connections.""" import asyncio import logging from typing import Dict, Set from opentrons.hardware_control.emulation.module_server.models import ( - Connection, + ModuleConnection, Message, ) from opentrons.hardware_control.emulation.proxy import ProxyListener @@ -16,7 +17,11 @@ class ModuleStatusServer(ProxyListener): - """Server notifying of module connections.""" + """The module status server is the emulator equivalent of inotify. A client + can know when an emulated module connects or disconnects. + + Clients connect and read JSON messages (See models module). + """ def __init__(self, settings: ModuleServerSettings) -> None: """Constructor @@ -25,7 +30,7 @@ def __init__(self, settings: ModuleServerSettings) -> None: settings: app settings """ self._settings = settings - self._connections: Dict[str, Connection] = {} + self._connections: Dict[str, ModuleConnection] = {} self._clients: Set[asyncio.StreamWriter] = set() def on_server_connected( @@ -42,7 +47,7 @@ def on_server_connected( """ log.info(f"On connected {server_type} {client_uri} {identifier}") - connection = Connection( + connection = ModuleConnection( module_type=server_type, url=client_uri, identifier=identifier ) self._connections[identifier] = connection @@ -87,6 +92,7 @@ async def _handle_connection( """Handle a client connection to the server.""" log.info("Client connected to module server.") + # A client connected. Send a dump of all connected modules. m = Message(status="dump", connections=list(self._connections.values())) writer.write(m.json().encode()) diff --git a/api/src/opentrons/hardware_control/emulation/proxy.py b/api/src/opentrons/hardware_control/emulation/proxy.py index 3ebc94a961a..0dfbb2e2eb6 100644 --- a/api/src/opentrons/hardware_control/emulation/proxy.py +++ b/api/src/opentrons/hardware_control/emulation/proxy.py @@ -1,3 +1,4 @@ +"""The Proxy class module.""" from __future__ import annotations import asyncio import logging @@ -14,7 +15,7 @@ @dataclass(frozen=True) class Connection: - """A connection.""" + """Attributes of a client connected on the server port (module emulator).""" identifier: str reader: asyncio.StreamReader @@ -22,6 +23,9 @@ class Connection: class ProxyListener(ABC): + """Interface defining an object needing to know when a server (module emulator) + connected/disconnected.""" + @abstractmethod def on_server_connected( self, server_type: str, client_uri: str, identifier: str diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py index a8852860db6..441917b8236 100644 --- a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py +++ b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py @@ -5,7 +5,7 @@ from opentrons.drivers.rpi_drivers.types import USBPort from opentrons.hardware_control.emulation.module_server import ( helpers, - ModuleServerClient, + ModuleStatusClient, ) from opentrons.hardware_control.emulation.module_server import models from opentrons.hardware_control.modules import ModuleAtPort @@ -20,7 +20,7 @@ def mock_callback() -> AsyncMock: @pytest.fixture def mock_client() -> AsyncMock: """Mock client.""" - return AsyncMock(spec=ModuleServerClient) + return AsyncMock(spec=ModuleStatusClient) @pytest.fixture @@ -30,10 +30,10 @@ def subject(mock_callback: AsyncMock, mock_client: AsyncMock) -> helpers.ModuleL @pytest.fixture -def connections() -> List[models.Connection]: +def connections() -> List[models.ModuleConnection]: """Connection models.""" return [ - models.Connection( + models.ModuleConnection( url=f"url{i}", module_type=f"module_type{i}", identifier=f"identifier{i}" ) for i in range(5) @@ -65,7 +65,7 @@ async def test_handle_message_connected_empty( async def test_handle_message_connected_one( subject: helpers.ModuleListener, mock_callback: AsyncMock, - connections: List[models.Connection], + connections: List[models.ModuleConnection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to add.""" @@ -77,7 +77,7 @@ async def test_handle_message_connected_one( async def test_handle_message_connected_many( subject: helpers.ModuleListener, mock_callback: AsyncMock, - connections: List[models.Connection], + connections: List[models.ModuleConnection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to add.""" @@ -98,7 +98,7 @@ async def test_handle_message_disconnected_empty( async def test_handle_message_disconnected_one( subject: helpers.ModuleListener, mock_callback: AsyncMock, - connections: List[models.Connection], + connections: List[models.ModuleConnection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to remove.""" @@ -110,7 +110,7 @@ async def test_handle_message_disconnected_one( async def test_handle_message_disconnected_many( subject: helpers.ModuleListener, mock_callback: AsyncMock, - connections: List[models.Connection], + connections: List[models.ModuleConnection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to remove.""" @@ -131,7 +131,7 @@ async def test_handle_message_dump_empty( async def test_handle_message_dump_one( subject: helpers.ModuleListener, mock_callback: AsyncMock, - connections: List[models.Connection], + connections: List[models.ModuleConnection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to load.""" @@ -143,7 +143,7 @@ async def test_handle_message_dump_one( async def test_handle_message_dump_many( subject: helpers.ModuleListener, mock_callback: AsyncMock, - connections: List[models.Connection], + connections: List[models.ModuleConnection], modules_at_port: List[ModuleAtPort], ) -> None: """It should call the call back with the correct modules to load.""" diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 6590766fa34..742413da920 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -4,7 +4,7 @@ import pytest import asyncio -from opentrons.hardware_control.emulation.module_server import ModuleServerClient +from opentrons.hardware_control.emulation.module_server import ModuleStatusClient from opentrons.hardware_control.emulation.module_server.helpers import wait_emulators from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control.emulation.types import ModuleType @@ -35,7 +35,7 @@ def _run_app() -> None: asyncio.run(run_app.run(emulator_settings, modules=[m.value for m in modules])) async def _wait_ready() -> None: - c = await ModuleServerClient.connect( + c = await ModuleStatusClient.connect( host="localhost", port=emulator_settings.module_server.port, interval_seconds=1, From 26db3e21db53a6e95d865a50e362918642d75173 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Fri, 29 Oct 2021 11:08:31 -0400 Subject: [PATCH 19/19] clean up. --- g-code-testing/g_code_parsing/g_code_engine.py | 4 ++-- .../g_code_test_data/http/http_settings.py | 8 +++++--- .../protocol/protocol_configurations.py | 18 +++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index 06aed89b099..a1443b85f5d 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -15,7 +15,7 @@ from opentrons.config.robot_configs import build_config from opentrons.hardware_control.emulation.module_server.helpers import ( wait_emulators, - ModuleServerClient, + ModuleStatusClient, ) from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control import API, ThreadManager @@ -76,7 +76,7 @@ def _run_app(): # Entry point for process that waits for emulation to be ready. async def _wait_ready() -> None: - c = await ModuleServerClient.connect( + c = await ModuleStatusClient.connect( host="localhost", port=self._config.module_server.port ) await wait_emulators(client=c, modules=modules, timeout=5) diff --git a/g-code-testing/g_code_test_data/http/http_settings.py b/g-code-testing/g_code_test_data/http/http_settings.py index 787f265b4e0..15e5008d0d4 100644 --- a/g-code-testing/g_code_test_data/http/http_settings.py +++ b/g-code-testing/g_code_test_data/http/http_settings.py @@ -1,9 +1,11 @@ -from opentrons.hardware_control.emulation.settings import Settings, SmoothieSettings +from opentrons.hardware_control.emulation.settings import ( + Settings, SmoothieSettings, PipetteSettings +) HTTP_SETTINGS = Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_single_v2.1", "id": "P20SV202020070101"} + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101") ) ) diff --git a/g-code-testing/g_code_test_data/protocol/protocol_configurations.py b/g-code-testing/g_code_test_data/protocol/protocol_configurations.py index 3b889b2c164..bd55e48dcbd 100644 --- a/g-code-testing/g_code_test_data/protocol/protocol_configurations.py +++ b/g-code-testing/g_code_test_data/protocol/protocol_configurations.py @@ -1,4 +1,4 @@ -from opentrons.hardware_control.emulation.settings import Settings, SmoothieSettings +from opentrons.hardware_control.emulation.settings import Settings, SmoothieSettings, PipetteSettings from g_code_test_data.g_code_configuration import ProtocolGCodeConfirmConfig import pytest @@ -8,8 +8,8 @@ SWIFT_SMOOTHIE_SETTINGS = Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_multi_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_multi_v2.1", id="P20SV202020070101"), ) ) @@ -22,8 +22,8 @@ path="protocol/protocols/smoothie_protocol.py", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), ) ) ) @@ -33,8 +33,8 @@ path="protocol/protocols/2_single_channel_v2.py", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_single_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101"), ) ) ) @@ -44,8 +44,8 @@ path="protocol/protocols/2_modules_1s_1m_v2.py", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p300_single_v2.1", "id": "P20SV202020070101"}, - right={"model": "p20_multi_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101"), + right=PipetteSettings(model="p20_multi_v2.1", id="P20SV202020070101"), ) ) )