From d79d8b3453eac02b1e1c17e8eb718b4483faf208 Mon Sep 17 00:00:00 2001 From: amitlissack Date: Thu, 4 Nov 2021 13:38:53 -0400 Subject: [PATCH] refactor(emulation): integrated with module control (#8586) * Module control refactors * new package. * start module control integration. * tests for module control integration. * implement connection listener. * module server integrated into module control. * lint * g-code-testing * redo docker compose to deal with separate emulator apps. * usb port. * expand module emulation settings. * update docker readme. * format-js * lint * go back to threadings. I don't want to debug windows nonsense. * fix bug. * redo gcode testing's emulator setup. * documentation. * clean up. --- DOCKER.md | 56 +++++ api/src/opentrons/hardware_control/api.py | 8 +- .../opentrons/hardware_control/controller.py | 3 +- .../hardware_control/emulation/magdeck.py | 17 +- .../emulation/module_server.py | 216 ------------------ .../emulation/module_server/__init__.py | 8 + .../emulation/module_server/client.py | 72 ++++++ .../emulation/module_server/helpers.py | 129 +++++++++++ .../emulation/module_server/models.py | 31 +++ .../emulation/module_server/server.py | 108 +++++++++ .../hardware_control/emulation/proxy.py | 8 +- .../emulation/scripts/run_module_emulator.py | 8 +- .../hardware_control/emulation/settings.py | 47 +++- .../hardware_control/emulation/tempdeck.py | 24 +- .../emulation/thermocycler.py | 25 +- .../hardware_control/emulation/types.py | 4 +- .../hardware_control/module_control.py | 79 +++---- .../opentrons/hardware_control/simulator.py | 2 - .../emulation/module_server/__init__.py | 0 .../emulation/module_server/test_helpers.py | 152 ++++++++++++ .../hardware_control/integration/conftest.py | 44 ++-- docker-compose.yml | 35 ++- g-code-testing/cli.py | 2 +- .../g_code_parsing/g_code_engine.py | 107 ++++----- .../g_code_test_data/g_code_configuration.py | 4 +- .../g_code_test_data/http/http_settings.py | 8 +- .../g_code_test_data/http/modules/magdeck.py | 8 +- .../g_code_test_data/http/modules/tempdeck.py | 8 +- .../http/modules/thermocycler.py | 17 +- .../protocol/protocol_configurations.py | 18 +- .../g_code_parsing/test_g_code_engine.py | 1 - 31 files changed, 834 insertions(+), 415 deletions(-) delete mode 100644 api/src/opentrons/hardware_control/emulation/module_server.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/__init__.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/client.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/helpers.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/models.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/server.py create mode 100644 api/tests/opentrons/hardware_control/emulation/module_server/__init__.py create mode 100644 api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py diff --git a/DOCKER.md b/DOCKER.md index 6a02c7f459e8..3bb4026b34f9 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -37,6 +37,62 @@ For example to use a `p300_multi` on the right add: OT_EMULATOR_smoothie: '{"right": {"model": "p300_multi"}}' ``` +### Adding more emulators + +#### Magdeck + +To add a second mag deck emulator make a copy of the existing `magdeck` section and change the key and `serial_number`. + +For example this adds a `magdeck` with the serial number `magdeck2`: + +``` + magdeck2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator magdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_magdeck: '{"serial_number": "magdeck2", "model":"mag_deck_v20", "version":"2.0.0"}' +``` + +#### Tempdeck + +To add a second temp deck emulator make a copy of the existing `tempdeck` section and change the key and `serial_number`. + +For example this adds a `tempdeck` with the serial number `tempdeck2`: + +``` + tempdeck2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator tempdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_tempdeck: '{"serial_number": "tempdeck2", "model":"temp_deck_v20", "version":"v2.0.1", "temperature": {"starting":0.0, "degrees_per_tick": 2.0}}' +``` + +#### Thermocycler + +To add a second thermocycler emulator make a copy of the existing `thermocycler` section and change the key and `serial_number`. + +For example this adds a `thermocycler` with the serial number `thermocycler2`: + +``` + thermocycler2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator thermocycler emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_thermocycler: '{"serial_number": "thermocycler2", "model":"v02", "version":"v1.1.0", "lid_temperature": {"starting":23.0, "degrees_per_tick": 2.0}, "plate_temperature": {"starting":23.0, "degrees_per_tick": 2.0}}' +``` + ## Known Issues - Pipettes cannot be changed at run time. diff --git a/api/src/opentrons/hardware_control/api.py b/api/src/opentrons/hardware_control/api.py index 89f30cf80bd5..3ce5a0317a5f 100644 --- a/api/src/opentrons/hardware_control/api.py +++ b/api/src/opentrons/hardware_control/api.py @@ -214,7 +214,9 @@ async def blink(): api_instance = cls(backend, loop=checked_loop, config=checked_config) await api_instance.cache_instruments() - module_controls = await AttachedModulesControl.build(api_instance) + module_controls = await AttachedModulesControl.build( + api_instance, board_revision=backend.board_revision + ) backend.module_controls = module_controls checked_loop.create_task(backend.watch(loop=checked_loop)) backend.start_gpio_door_watcher( @@ -260,7 +262,9 @@ async def build_hardware_simulator( ) api_instance = cls(backend, loop=checked_loop, config=checked_config) await api_instance.cache_instruments() - module_controls = await AttachedModulesControl.build(api_instance) + module_controls = await AttachedModulesControl.build( + api_instance, board_revision=backend.board_revision + ) backend.module_controls = module_controls await backend.watch() return api_instance diff --git a/api/src/opentrons/hardware_control/controller.py b/api/src/opentrons/hardware_control/controller.py index 909ffaf5e64a..e63be8887312 100644 --- a/api/src/opentrons/hardware_control/controller.py +++ b/api/src/opentrons/hardware_control/controller.py @@ -11,7 +11,7 @@ aionotify = None from opentrons.drivers.smoothie_drivers import SmoothieDriver -from opentrons.drivers.rpi_drivers import build_gpio_chardev, usb +from opentrons.drivers.rpi_drivers import build_gpio_chardev import opentrons.config from opentrons.config import pipette_config from opentrons.config.types import RobotConfig @@ -77,7 +77,6 @@ def __init__(self, config: RobotConfig, gpio: GPIODriverLike): config=self.config, gpio_chardev=self._gpio_chardev ) self._cached_fw_version: Optional[str] = None - self._usb = usb.USBBus(self._board_revision) self._module_controls: Optional[AttachedModulesControl] = None try: self._event_watcher = self._build_event_watcher() diff --git a/api/src/opentrons/hardware_control/emulation/magdeck.py b/api/src/opentrons/hardware_control/emulation/magdeck.py index bfcc8d395871..eaa7eb718d1c 100644 --- a/api/src/opentrons/hardware_control/emulation/magdeck.py +++ b/api/src/opentrons/hardware_control/emulation/magdeck.py @@ -8,24 +8,21 @@ from opentrons.drivers.mag_deck.driver import GCODE from opentrons.hardware_control.emulation.parser import Parser, Command from .abstract_emulator import AbstractEmulator +from .settings import MagDeckSettings logger = logging.getLogger(__name__) -SERIAL = "magnetic_emulator" -MODEL = "mag_deck_v20" -VERSION = "2.0.0" - - class MagDeckEmulator(AbstractEmulator): """Magdeck emulator""" height: float = 0 position: float = 0 - def __init__(self, parser: Parser) -> None: - self.reset() + def __init__(self, parser: Parser, settings: MagDeckSettings) -> None: + self._settings = settings self._parser = parser + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -53,7 +50,11 @@ def _handle(self, command: Command) -> Optional[str]: elif command.gcode == GCODE.GET_CURRENT_POSITION: return f"Z:{self.position}" elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) elif command.gcode == GCODE.PROGRAMMING_MODE: pass return None diff --git a/api/src/opentrons/hardware_control/emulation/module_server.py b/api/src/opentrons/hardware_control/emulation/module_server.py deleted file mode 100644 index 2a09a21afd2f..000000000000 --- a/api/src/opentrons/hardware_control/emulation/module_server.py +++ /dev/null @@ -1,216 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging - -from opentrons.hardware_control.emulation.types import ModuleType -from typing_extensions import Literal, Final -from typing import Dict, List, Set, Sequence, Optional -from pydantic import BaseModel - -from opentrons.hardware_control.emulation.proxy import ProxyListener -from opentrons.hardware_control.emulation.settings import ModuleServerSettings - - -log = logging.getLogger(__name__) - -MessageDelimiter: Final = b"\n" - - -class ModuleStatusServer(ProxyListener): - """Server notifying of module connections.""" - - def __init__(self, settings: ModuleServerSettings) -> None: - """Constructor - - Args: - settings: app settings - """ - self._settings = settings - self._connections: Dict[str, Connection] = {} - self._clients: Set[asyncio.StreamWriter] = set() - - def on_server_connected( - self, server_type: str, client_uri: str, identifier: str - ) -> None: - """Called when a new module has connected. - - Args: - server_type: the type of module - client_uri: the url string for a driver to connect to - identifier: unique id for connection - - Returns: None - - """ - log.info(f"On connected {server_type} {client_uri} {identifier}") - connection = Connection( - module_type=server_type, url=client_uri, identifier=identifier - ) - self._connections[identifier] = connection - for c in self._clients: - c.write( - Message(status="connected", connections=[connection]).json().encode() - ) - c.write(b"\n") - - def on_server_disconnected(self, identifier: str) -> None: - """Called when a module has disconnected. - - Args: - identifier: unique id for the connection - - Returns: None - """ - log.info(f"On disconnected {identifier}") - try: - connection = self._connections[identifier] - del self._connections[identifier] - for c in self._clients: - c.write( - Message(status="disconnected", connections=[connection]) - .json() - .encode() - ) - c.write(MessageDelimiter) - except KeyError: - log.exception("Failed to find identifier") - - async def run(self) -> None: - """Run the server.""" - server = await asyncio.start_server( - self._handle_connection, host=self._settings.host, port=self._settings.port - ) - await server.serve_forever() - - async def _handle_connection( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter - ) -> None: - """Handle a client connection to the server.""" - log.info("Client connected to module server.") - - m = Message(status="dump", connections=list(self._connections.values())) - - writer.write(m.json().encode()) - writer.write(MessageDelimiter) - - self._clients.add(writer) - - while True: - if b"" == await reader.read(): - self._clients.remove(writer) - break - - log.info("Client disconnected from module server.") - - -class Connection(BaseModel): - """Model a single module connection.""" - - url: str - module_type: str - identifier: str - - -class Message(BaseModel): - """A message sent to module server clients.""" - - status: Literal["connected", "disconnected", "dump"] - connections: List[Connection] - - -class ModuleServerClient: - """A module server client.""" - - def __init__( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter - ) -> None: - """Constructor.""" - self._reader = reader - self._writer = writer - - @classmethod - async def connect( - cls, - host: str, - port: int, - retries: int = 3, - interval_seconds: float = 0.1, - ) -> ModuleServerClient: - """Connect to the module server. - - Args: - host: module server host. - port: module server port. - retries: number of retries - interval_seconds: time between retries. - - Returns: - None - Raises: - IOError on retry expiry. - """ - r: Optional[asyncio.StreamReader] = None - w: Optional[asyncio.StreamWriter] = None - for i in range(retries): - try: - r, w = await asyncio.open_connection(host=host, port=port) - except OSError: - await asyncio.sleep(interval_seconds) - - if r is not None and w is not None: - return ModuleServerClient(reader=r, writer=w) - else: - raise IOError( - f"Failed to connect to module_server at after {retries} retries." - ) - - async def read(self) -> Message: - """Read a message from the module server.""" - b = await self._reader.readuntil(MessageDelimiter) - m: Message = Message.parse_raw(b) - return m - - def close(self) -> None: - """Close the client.""" - self._writer.close() - - -async def wait_emulators( - client: ModuleServerClient, - modules: Sequence[ModuleType], - timeout: float, -) -> None: - """Wait for module emulators to connect. - - Args: - client: module server client. - modules: collection of of module types to wait for. - timeout: how long to wait for emulators to connect (in seconds) - - Returns: - None - Raises: - asyncio.TimeoutError on timeout. - """ - - async def _wait_modules(cl: ModuleServerClient, module_set: Set[str]) -> None: - """Read messages from module server waiting for modules in module_set to - be connected.""" - while module_set: - m: Message = await cl.read() - if m.status == "dump" or m.status == "connected": - for c in m.connections: - if c.module_type in module_set: - module_set.remove(c.module_type) - elif m.status == "disconnected": - for c in m.connections: - if c.module_type in module_set: - module_set.add(c.module_type) - - log.debug(f"after message: {m}, awaiting module set is: {module_set}") - - await asyncio.wait_for( - _wait_modules(cl=client, module_set=set(n.value for n in modules)), - timeout=timeout, - ) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/__init__.py b/api/src/opentrons/hardware_control/emulation/module_server/__init__.py new file mode 100644 index 000000000000..082295dbfc76 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/__init__.py @@ -0,0 +1,8 @@ +"""Package for the module status server.""" +from .server import ModuleStatusServer +from .client import ModuleStatusClient + +__all__ = [ + "ModuleStatusServer", + "ModuleStatusClient", +] diff --git a/api/src/opentrons/hardware_control/emulation/module_server/client.py b/api/src/opentrons/hardware_control/emulation/module_server/client.py new file mode 100644 index 000000000000..b1051861fc67 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/client.py @@ -0,0 +1,72 @@ +from __future__ import annotations +import asyncio +from asyncio import IncompleteReadError, LimitOverrunError +from typing import Optional + +from opentrons.hardware_control.emulation.module_server.models import Message +from opentrons.hardware_control.emulation.module_server.server import MessageDelimiter + + +class ModuleServerClientError(Exception): + pass + + +class ModuleStatusClient: + """A module server client.""" + + def __init__( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """Constructor.""" + self._reader = reader + self._writer = writer + + @classmethod + async def connect( + cls, + host: str, + port: int, + retries: int = 3, + interval_seconds: float = 0.1, + ) -> ModuleStatusClient: + """Connect to the module server. + + Args: + host: module server host. + port: module server port. + retries: number of retries + interval_seconds: time between retries. + + Returns: + None + Raises: + IOError on retry expiry. + """ + r: Optional[asyncio.StreamReader] = None + w: Optional[asyncio.StreamWriter] = None + for i in range(retries): + try: + r, w = await asyncio.open_connection(host=host, port=port) + break + except OSError: + await asyncio.sleep(interval_seconds) + + if r is not None and w is not None: + return ModuleStatusClient(reader=r, writer=w) + else: + raise IOError( + f"Failed to connect to module_server at after {retries} retries." + ) + + async def read(self) -> Message: + """Read a message from the module server.""" + try: + b = await self._reader.readuntil(MessageDelimiter) + m: Message = Message.parse_raw(b) + return m + except (IncompleteReadError, LimitOverrunError) as e: + raise ModuleServerClientError(str(e)) + + def close(self) -> None: + """Close the client.""" + self._writer.close() diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py new file mode 100644 index 000000000000..ad4e7ffbabeb --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -0,0 +1,129 @@ +"""Utililty methods and classes for interacting with the Module Status Server.""" + +import asyncio +from typing import Sequence, Set, Callable, List, Awaitable + +from opentrons.drivers.rpi_drivers.types import USBPort +from opentrons.hardware_control.emulation.module_server.client import ( + ModuleStatusClient, + ModuleServerClientError, +) +from opentrons.hardware_control.emulation.module_server.models import Message +from opentrons.hardware_control.emulation.module_server.server import log +from opentrons.hardware_control.emulation.settings import Settings +from opentrons.hardware_control.emulation.types import ModuleType +from opentrons.hardware_control.modules import ModuleAtPort + +NotifyMethod = Callable[[List[ModuleAtPort], List[ModuleAtPort]], Awaitable[None]] +"""Signature of method to be notified of new and removed modules.""" + + +async def listen_module_connection(callback: NotifyMethod) -> None: + """Listen for module emulator connections.""" + settings = Settings() + try: + client = await ModuleStatusClient.connect( + host=settings.module_server.host, + port=settings.module_server.port, + interval_seconds=1.0, + ) + listener = ModuleListener(client=client, notify_method=callback) + await listener.run() + except IOError: + log.exception("Failed to connect to module server.") + + +class ModuleListener: + """Provide a callback for listening for new and removed module connections.""" + + def __init__(self, client: ModuleStatusClient, notify_method: NotifyMethod) -> None: + """Constructor. + + Args: + client: A module server client + notify_method: callback method. + + Returns: + None + """ + self._client = client + self._notify_method = notify_method + self._hub_index = 1 + + async def run(self) -> None: + """Run the listener.""" + while True: + try: + m = await self._client.read() + await self.handle_message(message=m) + except ModuleServerClientError: + log.exception("Read error.") + break + + async def handle_message(self, message: Message) -> None: + """Call callback with results of message. + + Args: + message: Message object from module server + + Returns: + None + """ + + def _next_index() -> int: + index = self._hub_index + self._hub_index += 1 + return index + + connections = [ + ModuleAtPort( + port=c.url, + name=c.module_type, + usb_port=USBPort(name=c.identifier, sub_names=[], hub=_next_index()), + ) + for c in message.connections + ] + if message.status == "connected" or message.status == "dump": + await self._notify_method(connections, []) + elif message.status == "disconnected": + await self._notify_method([], connections) + + +async def wait_emulators( + client: ModuleStatusClient, + modules: Sequence[ModuleType], + timeout: float, +) -> None: + """Wait for module emulators to connect. + + Args: + client: module server client. + modules: collection of of module types to wait for. + timeout: how long to wait for emulators to connect (in seconds) + + Returns: + None + Raises: + asyncio.TimeoutError on timeout. + """ + + async def _wait_modules(cl: ModuleStatusClient, module_set: Set[str]) -> None: + """Read messages from module server waiting for modules in module_set to + be connected.""" + while module_set: + m: Message = await cl.read() + if m.status == "dump" or m.status == "connected": + for c in m.connections: + if c.module_type in module_set: + module_set.remove(c.module_type) + elif m.status == "disconnected": + for c in m.connections: + if c.module_type in module_set: + module_set.add(c.module_type) + + log.debug(f"after message: {m}, awaiting module set is: {module_set}") + + await asyncio.wait_for( + _wait_modules(cl=client, module_set=set(n.value for n in modules)), + timeout=timeout, + ) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/models.py b/api/src/opentrons/hardware_control/emulation/module_server/models.py new file mode 100644 index 000000000000..05363bfc109e --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/models.py @@ -0,0 +1,31 @@ +from typing import List + +from typing_extensions import Literal + +from pydantic import BaseModel, Field + + +class ModuleConnection(BaseModel): + """Model a single module connection.""" + + url: str = Field( + ..., + description="The url (port) value the module driver should connect to. " + "For example: socket://host:port", + ) + module_type: str = Field( + ..., description="What kind of module this connection emulates." + ) + identifier: str = Field(..., description="Unique id for this emulator.") + + +class Message(BaseModel): + """A message sent to module server clients.""" + + status: Literal["connected", "disconnected", "dump"] = Field( + ..., + description="`dump` includes a complete list of connected emulators. " + "`connected` for new connections. `disconnected` for emulators " + "that have disconnected. ", + ) + connections: List[ModuleConnection] diff --git a/api/src/opentrons/hardware_control/emulation/module_server/server.py b/api/src/opentrons/hardware_control/emulation/module_server/server.py new file mode 100644 index 000000000000..5a3d696eb7b8 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/server.py @@ -0,0 +1,108 @@ +"""Server notifying of module connections.""" +import asyncio +import logging +from typing import Dict, Set + +from opentrons.hardware_control.emulation.module_server.models import ( + ModuleConnection, + Message, +) +from opentrons.hardware_control.emulation.proxy import ProxyListener +from opentrons.hardware_control.emulation.settings import ModuleServerSettings +from typing_extensions import Final + +log = logging.getLogger(__name__) + +MessageDelimiter: Final = b"\n" + + +class ModuleStatusServer(ProxyListener): + """The module status server is the emulator equivalent of inotify. A client + can know when an emulated module connects or disconnects. + + Clients connect and read JSON messages (See models module). + """ + + def __init__(self, settings: ModuleServerSettings) -> None: + """Constructor + + Args: + settings: app settings + """ + self._settings = settings + self._connections: Dict[str, ModuleConnection] = {} + self._clients: Set[asyncio.StreamWriter] = set() + + def on_server_connected( + self, server_type: str, client_uri: str, identifier: str + ) -> None: + """Called when a new module has connected. + + Args: + server_type: the type of module + client_uri: the url string for a driver to connect to + identifier: unique id for connection + + Returns: None + + """ + log.info(f"On connected {server_type} {client_uri} {identifier}") + connection = ModuleConnection( + module_type=server_type, url=client_uri, identifier=identifier + ) + self._connections[identifier] = connection + for c in self._clients: + c.write( + Message(status="connected", connections=[connection]).json().encode() + ) + c.write(b"\n") + + def on_server_disconnected(self, identifier: str) -> None: + """Called when a module has disconnected. + + Args: + identifier: unique id for the connection + + Returns: None + """ + log.info(f"On disconnected {identifier}") + try: + connection = self._connections[identifier] + del self._connections[identifier] + for c in self._clients: + c.write( + Message(status="disconnected", connections=[connection]) + .json() + .encode() + ) + c.write(MessageDelimiter) + except KeyError: + log.exception("Failed to find identifier") + + async def run(self) -> None: + """Run the server.""" + server = await asyncio.start_server( + self._handle_connection, host=self._settings.host, port=self._settings.port + ) + await server.serve_forever() + + async def _handle_connection( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """Handle a client connection to the server.""" + log.info("Client connected to module server.") + + # A client connected. Send a dump of all connected modules. + m = Message(status="dump", connections=list(self._connections.values())) + + writer.write(m.json().encode()) + writer.write(MessageDelimiter) + + self._clients.add(writer) + + while True: + if b"" == await reader.read(): + self._clients.remove(writer) + break + + log.info("Client disconnected from module server.") diff --git a/api/src/opentrons/hardware_control/emulation/proxy.py b/api/src/opentrons/hardware_control/emulation/proxy.py index 6877c89ebf5a..0dfbb2e2eb68 100644 --- a/api/src/opentrons/hardware_control/emulation/proxy.py +++ b/api/src/opentrons/hardware_control/emulation/proxy.py @@ -1,3 +1,4 @@ +"""The Proxy class module.""" from __future__ import annotations import asyncio import logging @@ -14,7 +15,7 @@ @dataclass(frozen=True) class Connection: - """A connection.""" + """Attributes of a client connected on the server port (module emulator).""" identifier: str reader: asyncio.StreamReader @@ -22,6 +23,9 @@ class Connection: class ProxyListener(ABC): + """Interface defining an object needing to know when a server (module emulator) + connected/disconnected.""" + @abstractmethod def on_server_connected( self, server_type: str, client_uri: str, identifier: str @@ -115,7 +119,7 @@ async def _handle_server_connection( self._cons.append(connection) self._event_listener.on_server_connected( server_type=self._name, - client_uri=f"{socket.gethostname()}:{self._settings.driver_port}", + client_uri=f"socket://{socket.gethostname()}:{self._settings.driver_port}", identifier=connection.identifier, ) diff --git a/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py b/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py index 4c19fe539d29..e05ea632c98a 100644 --- a/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py @@ -17,9 +17,11 @@ from opentrons.hardware_control.emulation.settings import Settings, ProxySettings emulator_builder: Final[Dict[str, Callable[[Settings], AbstractEmulator]]] = { - ModuleType.Magnetic.value: lambda s: MagDeckEmulator(Parser()), - ModuleType.Temperature.value: lambda s: TempDeckEmulator(Parser()), - ModuleType.Thermocycler.value: lambda s: ThermocyclerEmulator(Parser()), + ModuleType.Magnetic.value: lambda s: MagDeckEmulator(Parser(), s.magdeck), + ModuleType.Temperature.value: lambda s: TempDeckEmulator(Parser(), s.tempdeck), + ModuleType.Thermocycler.value: lambda s: ThermocyclerEmulator( + Parser(), s.thermocycler + ), } emulator_port: Final[Dict[str, Callable[[Settings], ProxySettings]]] = { diff --git a/api/src/opentrons/hardware_control/emulation/settings.py b/api/src/opentrons/hardware_control/emulation/settings.py index 8e62cfc1192d..d81de05275fa 100644 --- a/api/src/opentrons/hardware_control/emulation/settings.py +++ b/api/src/opentrons/hardware_control/emulation/settings.py @@ -1,3 +1,4 @@ +from opentrons.hardware_control.emulation.util import TEMPERATURE_ROOM from pydantic import BaseSettings, BaseModel @@ -17,7 +18,31 @@ class SmoothieSettings(BaseModel): port: int = 9996 -class ProxySettings(BaseSettings): +class BaseModuleSettings(BaseModel): + serial_number: str + model: str + version: str + + +class TemperatureModelSettings(BaseModel): + degrees_per_tick: float = 2.0 + starting: float = float(TEMPERATURE_ROOM) + + +class MagDeckSettings(BaseModuleSettings): + pass + + +class TempDeckSettings(BaseModuleSettings): + temperature: TemperatureModelSettings + + +class ThermocyclerSettings(BaseModuleSettings): + lid_temperature: TemperatureModelSettings + plate_temperature: TemperatureModelSettings + + +class ProxySettings(BaseModel): """Settings for a proxy.""" host: str = "0.0.0.0" @@ -29,13 +54,27 @@ class ModuleServerSettings(BaseModel): """Settings for the module server""" host: str = "0.0.0.0" - port: int = 8888 + port: int = 8989 class Settings(BaseSettings): smoothie: SmoothieSettings = SmoothieSettings() - - host: str = "0.0.0.0" + magdeck: MagDeckSettings = MagDeckSettings( + serial_number="magnetic_emulator", model="mag_deck_v20", version="2.0.0" + ) + tempdeck: TempDeckSettings = TempDeckSettings( + serial_number="temperature_emulator", + model="temp_deck_v20", + version="v2.0.1", + temperature=TemperatureModelSettings(starting=0.0), + ) + thermocycler: ThermocyclerSettings = ThermocyclerSettings( + serial_number="thermocycler_emulator", + model="v02", + version="v1.1.0", + lid_temperature=TemperatureModelSettings(), + plate_temperature=TemperatureModelSettings(), + ) heatershaker_proxy: ProxySettings = ProxySettings( emulator_port=9000, driver_port=9995 diff --git a/api/src/opentrons/hardware_control/emulation/tempdeck.py b/api/src/opentrons/hardware_control/emulation/tempdeck.py index 9e99c1e23a84..22c237ee0327 100644 --- a/api/src/opentrons/hardware_control/emulation/tempdeck.py +++ b/api/src/opentrons/hardware_control/emulation/tempdeck.py @@ -9,6 +9,7 @@ from opentrons.drivers.temp_deck.driver import GCODE from opentrons.hardware_control.emulation import util from opentrons.hardware_control.emulation.parser import Parser, Command +from opentrons.hardware_control.emulation.settings import TempDeckSettings from .abstract_emulator import AbstractEmulator from .simulations import Temperature @@ -17,17 +18,15 @@ logger = logging.getLogger(__name__) -SERIAL = "temperature_emulator" -MODEL = "temp_deck_v20" -VERSION = "v2.0.1" - - class TempDeckEmulator(AbstractEmulator): """TempDeck emulator""" - def __init__(self, parser: Parser) -> None: - self.reset() + _temperature: Temperature + + def __init__(self, parser: Parser, settings: TempDeckSettings) -> None: + self._settings = settings self._parser = parser + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -36,7 +35,10 @@ def handle(self, line: str) -> Optional[str]: return None if not joined else joined def reset(self): - self._temperature = Temperature(per_tick=0.25, current=0.0) + self._temperature = Temperature( + per_tick=self._settings.temperature.degrees_per_tick, + current=self._settings.temperature.starting, + ) def _handle(self, command: Command) -> Optional[str]: """Handle a command.""" @@ -57,7 +59,11 @@ def _handle(self, command: Command) -> Optional[str]: elif command.gcode == GCODE.DISENGAGE: self._temperature.deactivate(util.TEMPERATURE_ROOM) elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) elif command.gcode == GCODE.PROGRAMMING_MODE: pass return None diff --git a/api/src/opentrons/hardware_control/emulation/thermocycler.py b/api/src/opentrons/hardware_control/emulation/thermocycler.py index f841b00adddc..e35f0d05214d 100644 --- a/api/src/opentrons/hardware_control/emulation/thermocycler.py +++ b/api/src/opentrons/hardware_control/emulation/thermocycler.py @@ -8,6 +8,7 @@ from opentrons.drivers.thermocycler.driver import GCODE from opentrons.drivers.types import ThermocyclerLidStatus from opentrons.hardware_control.emulation.parser import Parser, Command +from opentrons.hardware_control.emulation.settings import ThermocyclerSettings from .abstract_emulator import AbstractEmulator from .simulations import Temperature, TemperatureWithHold @@ -16,11 +17,6 @@ logger = logging.getLogger(__name__) -SERIAL = "thermocycler_emulator" -MODEL = "v02" -VERSION = "v1.1.0" - - class ThermocyclerEmulator(AbstractEmulator): """Thermocycler emulator""" @@ -30,9 +26,10 @@ class ThermocyclerEmulator(AbstractEmulator): plate_volume: util.OptionalValue[float] plate_ramp_rate: util.OptionalValue[float] - def __init__(self, parser: Parser) -> None: - self.reset() + def __init__(self, parser: Parser, settings: ThermocyclerSettings) -> None: self._parser = parser + self._settings = settings + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -41,9 +38,13 @@ def handle(self, line: str) -> Optional[str]: return None if not joined else joined def reset(self): - self._lid_temperature = Temperature(per_tick=2, current=util.TEMPERATURE_ROOM) + self._lid_temperature = Temperature( + per_tick=self._settings.lid_temperature.degrees_per_tick, + current=self._settings.lid_temperature.starting, + ) self._plate_temperature = TemperatureWithHold( - per_tick=2, current=util.TEMPERATURE_ROOM + per_tick=self._settings.plate_temperature.degrees_per_tick, + current=self._settings.plate_temperature.starting, ) self.lid_status = ThermocyclerLidStatus.OPEN self.plate_volume = util.OptionalValue[float]() @@ -115,7 +116,11 @@ def _handle(self, command: Command) -> Optional[str]: # noqa: C901 elif command.gcode == GCODE.DEACTIVATE_BLOCK: self._plate_temperature.deactivate(temperature=util.TEMPERATURE_ROOM) elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) return None @staticmethod diff --git a/api/src/opentrons/hardware_control/emulation/types.py b/api/src/opentrons/hardware_control/emulation/types.py index cbdd3eecb11c..1a5a7028ef34 100644 --- a/api/src/opentrons/hardware_control/emulation/types.py +++ b/api/src/opentrons/hardware_control/emulation/types.py @@ -4,7 +4,7 @@ class ModuleType(str, Enum): """Module type enumeration.""" - Magnetic = "magnetic" - Temperature = "temperature" + Magnetic = "magdeck" + Temperature = "tempdeck" Thermocycler = "thermocycler" Heatershaker = "heatershaker" diff --git a/api/src/opentrons/hardware_control/module_control.py b/api/src/opentrons/hardware_control/module_control.py index 9b63b70bfc0b..f7a5e43eb5e0 100644 --- a/api/src/opentrons/hardware_control/module_control.py +++ b/api/src/opentrons/hardware_control/module_control.py @@ -1,16 +1,16 @@ +from __future__ import annotations import logging import asyncio -import os import re from typing import List, Tuple, Optional from glob import glob from opentrons.config import IS_ROBOT, IS_LINUX -from opentrons.drivers.rpi_drivers import types -from opentrons.hardware_control.modules import ModuleAtPort - -from .execution_manager import ExecutionManager -from .types import AionotifyEvent +from opentrons.drivers.rpi_drivers import types, usb, usb_simulator +from opentrons.hardware_control.emulation.module_server.helpers import ( + listen_module_connection, +) +from .types import AionotifyEvent, BoardRevision from . import modules @@ -25,25 +25,32 @@ class AttachedModulesControl: USB port information and finally building a module object. """ - def __init__(self, api): + def __init__(self, api, board_revision: BoardRevision) -> None: self._available_modules: List[modules.AbstractModule] = [] self._api = api + self._usb = ( + usb.USBBus(board_revision) + if not api.is_simulator and IS_ROBOT + else usb_simulator.USBBusSimulator(board_revision) + ) @classmethod - async def build(cls, api_instance): - mc_instance = cls(api_instance) + async def build( + cls, api_instance, board_revision: BoardRevision + ) -> AttachedModulesControl: + mc_instance = cls(api_instance, board_revision) if not api_instance.is_simulator: await mc_instance.register_modules(mc_instance.scan()) + api_instance.loop.create_task( + listen_module_connection(mc_instance.register_modules) + ) + return mc_instance @property def available_modules(self) -> List[modules.AbstractModule]: return self._available_modules - @property - def api(self): - return self._api - async def build_module( self, port: str, @@ -56,9 +63,9 @@ async def build_module( port=port, usb_port=usb_port, which=model, - simulating=self.api.is_simulator, + simulating=self._api.is_simulator, loop=loop, - execution_manager=self.api._execution_manager, + execution_manager=self._api._execution_manager, sim_model=sim_model, ) @@ -107,14 +114,15 @@ async def register_modules( # destroy removed mods await self.unregister_modules(removed_mods_at_ports) - sorted_mods_at_port = self.api._backend._usb.match_virtual_ports( - new_mods_at_ports - ) + sorted_mods_at_port = self._usb.match_virtual_ports(new_mods_at_ports) # build new mods for mod in sorted_mods_at_port: new_instance = await self.build_module( - port=mod.port, usb_port=mod.usb_port, model=mod.name, loop=self.api.loop + port=mod.port, + usb_port=mod.usb_port, + model=mod.name, + loop=self._api.loop, ) self._available_modules.append(new_instance) log.info( @@ -145,7 +153,7 @@ async def parse_modules( for module in self.available_modules: if mod_type == module.name(): matching_modules.append(module) - if self.api.is_simulator: + if self._api.is_simulator: module_builder = { "magdeck": modules.MagDeck.build, "tempdeck": modules.TempDeck.build, @@ -154,10 +162,10 @@ async def parse_modules( if module_builder: simulating_module = await module_builder( port="", - usb_port=self.api._backend._usb.find_port(""), + usb_port=self._usb.find_port(""), simulating=True, - loop=self.api.loop, - execution_manager=ExecutionManager(loop=self.api.loop), + loop=self._api.loop, + execution_manager=self._api._execution_manager, sim_model=by_model.value, ) simulated_module = simulating_module @@ -180,21 +188,6 @@ def scan(self) -> List[modules.ModuleAtPort]: if module_at_port: discovered_modules.append(module_at_port) - # Check for emulator environment variables - emulator_uri = os.environ.get("OT_THERMOCYCLER_EMULATOR_URI") - if emulator_uri: - discovered_modules.append( - ModuleAtPort(port=emulator_uri, name="thermocycler") - ) - - emulator_uri = os.environ.get("OT_TEMPERATURE_EMULATOR_URI") - if emulator_uri: - discovered_modules.append(ModuleAtPort(port=emulator_uri, name="tempdeck")) - - emulator_uri = os.environ.get("OT_MAGNETIC_EMULATOR_URI") - if emulator_uri: - discovered_modules.append(ModuleAtPort(port=emulator_uri, name="magdeck")) - log.debug("Discovered modules: {}".format(discovered_modules)) return discovered_modules @@ -212,14 +205,16 @@ def get_module_at_port(port: str) -> Optional[modules.ModuleAtPort]: return modules.ModuleAtPort(port=f"/dev/{port}", name=name) return None - async def handle_module_appearance(self, event: AionotifyEvent): + async def handle_module_appearance(self, event: AionotifyEvent) -> None: """Only called upon availability of aionotify. Check that the file system has changed and either remove or add modules depending on the result. - :param event_name: The title of the even passed into aionotify. - :param event_flags: AionotifyFlags dataclass that maps flags listed from - the aionotify event. + Args: + event: The event passed from aionotify. + + Returns: + None """ maybe_module_at_port = self.get_module_at_port(event.name) new_modules = None diff --git a/api/src/opentrons/hardware_control/simulator.py b/api/src/opentrons/hardware_control/simulator.py index 8b07b263f5bc..684ab0e5bd1b 100644 --- a/api/src/opentrons/hardware_control/simulator.py +++ b/api/src/opentrons/hardware_control/simulator.py @@ -14,7 +14,6 @@ from opentrons.drivers.smoothie_drivers import SimulatingDriver from opentrons.drivers.rpi_drivers.gpio_simulator import SimulatingGPIOCharDev -from opentrons.drivers.rpi_drivers import usb_simulator from . import modules from .types import BoardRevision, Axis @@ -154,7 +153,6 @@ def _sanitize_attached_instrument( self._log = MODULE_LOG.getChild(repr(self)) self._strict_attached = bool(strict_attached_instruments) self._board_revision = BoardRevision.OG - self._usb = usb_simulator.USBBusSimulator(self._board_revision) # TODO (lc 05-12-2021) In a follow-up refactor that pulls the execution # manager responsbility into the controller/backend itself as opposed # to the hardware api controller. diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/__init__.py b/api/tests/opentrons/hardware_control/emulation/module_server/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py new file mode 100644 index 000000000000..441917b8236f --- /dev/null +++ b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py @@ -0,0 +1,152 @@ +from typing import List + +import pytest +from mock import AsyncMock +from opentrons.drivers.rpi_drivers.types import USBPort +from opentrons.hardware_control.emulation.module_server import ( + helpers, + ModuleStatusClient, +) +from opentrons.hardware_control.emulation.module_server import models +from opentrons.hardware_control.modules import ModuleAtPort + + +@pytest.fixture +def mock_callback() -> AsyncMock: + """Callback mock.""" + return AsyncMock(spec=helpers.NotifyMethod) + + +@pytest.fixture +def mock_client() -> AsyncMock: + """Mock client.""" + return AsyncMock(spec=ModuleStatusClient) + + +@pytest.fixture +def subject(mock_callback: AsyncMock, mock_client: AsyncMock) -> helpers.ModuleListener: + """Test subject.""" + return helpers.ModuleListener(client=mock_client, notify_method=mock_callback) + + +@pytest.fixture +def connections() -> List[models.ModuleConnection]: + """Connection models.""" + return [ + models.ModuleConnection( + url=f"url{i}", module_type=f"module_type{i}", identifier=f"identifier{i}" + ) + for i in range(5) + ] + + +@pytest.fixture +def modules_at_port() -> List[ModuleAtPort]: + """Connection models.""" + return [ + ModuleAtPort( + port=f"url{i}", + name=f"module_type{i}", + usb_port=USBPort(name=f"identifier{i}", sub_names=[], hub=i + 1), + ) + for i in range(5) + ] + + +async def test_handle_message_connected_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=[]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], []) + + +async def test_handle_message_connected_one( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=connections[:1]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with(modules_at_port[:1], []) + + +async def test_handle_message_connected_many( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=connections) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with(modules_at_port, []) + + +async def test_handle_message_disconnected_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=[]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], []) + + +async def test_handle_message_disconnected_one( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=connections[:1]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], modules_at_port[:1]) + + +async def test_handle_message_disconnected_many( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=connections) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], modules_at_port) + + +async def test_handle_message_dump_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=[]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], []) + + +async def test_handle_message_dump_one( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=connections[:1]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with(modules_at_port[:1], []) + + +async def test_handle_message_dump_many( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=connections) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with(modules_at_port, []) diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 3ac09b546405..742413da920a 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -1,8 +1,11 @@ from typing import Iterator +import threading import pytest -import threading import asyncio + +from opentrons.hardware_control.emulation.module_server import ModuleStatusClient +from opentrons.hardware_control.emulation.module_server.helpers import wait_emulators from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control.emulation.types import ModuleType from opentrons.hardware_control.emulation.settings import ( @@ -16,7 +19,6 @@ def emulator_settings() -> Settings: """Emulator settings""" return Settings( - host="0.0.0.0", smoothie=SmoothieSettings( left=PipetteSettings(model="p20_multi_v2.0", id="P3HMV202020041605"), right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), @@ -27,24 +29,32 @@ def emulator_settings() -> Settings: @pytest.fixture(scope="session") def emulation_app(emulator_settings: Settings) -> Iterator[None]: """Run the emulators""" + modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] + + def _run_app() -> None: + asyncio.run(run_app.run(emulator_settings, modules=[m.value for m in modules])) - def runit() -> None: - asyncio.run( - run_app.run( - settings=emulator_settings, - modules=[ - ModuleType.Magnetic, - ModuleType.Temperature, - ModuleType.Thermocycler, - ], - ) + async def _wait_ready() -> None: + c = await ModuleStatusClient.connect( + host="localhost", + port=emulator_settings.module_server.port, + interval_seconds=1, ) + await wait_emulators(client=c, modules=modules, timeout=5) + c.close() - # TODO 20210219 - # The emulators must be run in a separate thread because our serial - # drivers block the main thread. Remove this thread when that is no - # longer true. - t = threading.Thread(target=runit) + def _run_wait_ready() -> None: + asyncio.run(_wait_ready()) + + # Start the emulator thread. + t = threading.Thread(target=_run_app) t.daemon = True t.start() + + # Start the wait for emulator ready thread and wait for it to terminate. + ready_proc = threading.Thread(target=_run_wait_ready) + ready_proc.daemon = True + ready_proc.start() + ready_proc.join() + yield diff --git a/docker-compose.yml b/docker-compose.yml index b66e1846fd47..931d70f49d94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,12 @@ services: build: . command: python3 -m opentrons.hardware_control.emulation.app ports: + - '8888:8888' + - '9000:9000' + - '9002:9002' + - '9003:9003' + - '9004:9004' + - '9995:9995' - '9996:9996' - '9997:9997' - '9998:9998' @@ -17,14 +23,33 @@ services: ports: - '31950:31950' environment: - - OT_API_CONFIG_DIR=/config - - OT_SMOOTHIE_EMULATOR_URI=socket://emulator:9996 - - OT_THERMOCYCLER_EMULATOR_URI=socket://emulator:9997 - - OT_TEMPERATURE_EMULATOR_URI=socket://emulator:9998 - - OT_MAGNETIC_EMULATOR_URI=socket://emulator:9999 + OT_API_CONFIG_DIR: /config + OT_SMOOTHIE_EMULATOR_URI: socket://emulator:9996 + OT_EMULATOR_module_server: '{"host": "emulator"}' links: - 'emulator' depends_on: - 'emulator' volumes: - .opentrons_config:/config:rw + tempdeck: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator tempdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + thermocycler: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator thermocycler emulator + links: + - 'emulator' + depends_on: + - 'emulator' + magdeck: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator magdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' diff --git a/g-code-testing/cli.py b/g-code-testing/cli.py index d0481f3b346d..8cb73ae1c578 100644 --- a/g-code-testing/cli.py +++ b/g-code-testing/cli.py @@ -78,7 +78,7 @@ def _diff(self) -> str: else: text = "No difference between compared strings" - return text + return text def _configurations(self) -> str: """Get a list of runnable G-Code Configurations""" diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index 36c323cb617d..a1443b85f5d2 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -1,8 +1,9 @@ import os import sys import asyncio +import time from multiprocessing import Process -from typing import Generator, Callable +from typing import Generator, Callable, Iterator from collections import namedtuple from opentrons.hardware_control.emulation.settings import Settings @@ -12,7 +13,10 @@ from contextlib import contextmanager from opentrons.protocol_api import ProtocolContext from opentrons.config.robot_configs import build_config -from opentrons.hardware_control.emulation import module_server +from opentrons.hardware_control.emulation.module_server.helpers import ( + wait_emulators, + ModuleStatusClient, +) from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control import API, ThreadManager from g_code_parsing.g_code_program.g_code_program import ( @@ -45,7 +49,6 @@ class GCodeEngine: def __init__(self, emulator_settings: Settings) -> None: self._config = emulator_settings - self._set_env_vars(emulator_settings) @staticmethod def _get_loop() -> asyncio.AbstractEventLoop: @@ -57,56 +60,52 @@ def _get_loop() -> asyncio.AbstractEventLoop: asyncio.set_event_loop(_loop) return asyncio.get_event_loop() - @staticmethod - def _set_env_vars(settings: Settings) -> None: - """Set URLs of where to find modules and config for smoothie""" - os.environ["OT_MAGNETIC_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.magdeck_proxy.driver_port - ) - os.environ["OT_THERMOCYCLER_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.thermocycler_proxy.driver_port - ) - os.environ["OT_TEMPERATURE_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.temperature_proxy.driver_port - ) - - @staticmethod - def _start_emulation_app(emulator_settings: Settings) -> Process: - """Start emulated OT-2""" + @contextmanager + def _emulate(self) -> Iterator[ThreadManager]: + """Context manager that starts emulated OT-2 hardware environment. A + hardware controller is returned.""" modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] - def runit(): - asyncio.run( - run_app.run(emulator_settings, modules=[m.value for m in modules]) - ) + # Entry point for the emulator app process + def _run_app(): + asyncio.run(run_app.run(self._config, modules=[m.value for m in modules])) - proc = Process(target=runit) + proc = Process(target=_run_app) proc.daemon = True proc.start() + # Entry point for process that waits for emulation to be ready. async def _wait_ready() -> None: - c = await module_server.ModuleServerClient.connect( - host="localhost", port=emulator_settings.module_server.port + c = await ModuleStatusClient.connect( + host="localhost", port=self._config.module_server.port ) - await module_server.wait_emulators(client=c, modules=modules, timeout=5) + await wait_emulators(client=c, modules=modules, timeout=5) c.close() - proc2 = Process(target=lambda: asyncio.run(_wait_ready())) - proc2.start() - proc2.join() + def _run_wait_ready(): + asyncio.run(_wait_ready()) - return proc + ready_proc = Process(target=_run_wait_ready) + ready_proc.daemon = True + ready_proc.start() + ready_proc.join() - @staticmethod - def _emulate_hardware(settings: Settings) -> ThreadManager: - """Created emulated smoothie""" + # Hardware controller conf = build_config({}) emulator = ThreadManager( API.build_hardware_controller, conf, - GCodeEngine.URI_TEMPLATE % settings.smoothie.port, + GCodeEngine.URI_TEMPLATE % self._config.smoothie.port, ) - return emulator + # Wait for modules to be present + while len(emulator.attached_modules) != len(modules): + time.sleep(0.1) + + yield emulator + + # Finished. Stop the emulator + proc.kill() + proc.join() @staticmethod def _get_protocol(file_path: str) -> Protocol: @@ -125,20 +124,16 @@ def run_protocol(self, path: str) -> Generator: :return: GCodeProgram with all the parsed data """ file_path = os.path.join(get_configuration_dir(), path) - app_process = self._start_emulation_app(emulator_settings=self._config) - protocol = self._get_protocol(file_path) - context = ProtocolContext( - implementation=ProtocolContextImplementation( - hardware=self._emulate_hardware(settings=self._config) - ), - loop=self._get_loop(), - ) - parsed_protocol = parse(protocol.text, protocol.filename) - with GCodeWatcher(emulator_settings=self._config) as watcher: - execute.run_protocol(parsed_protocol, context=context) - yield GCodeProgram.from_g_code_watcher(watcher) - app_process.terminate() - app_process.join() + with self._emulate() as h: + protocol = self._get_protocol(file_path) + context = ProtocolContext( + implementation=ProtocolContextImplementation(hardware=h), + loop=self._get_loop(), + ) + parsed_protocol = parse(protocol.text, protocol.filename) + with GCodeWatcher(emulator_settings=self._config) as watcher: + execute.run_protocol(parsed_protocol, context=context) + yield GCodeProgram.from_g_code_watcher(watcher) @contextmanager def run_http(self, executable: Callable): @@ -147,11 +142,7 @@ def run_http(self, executable: Callable): :param executable: Function connected to HTTP Request to execute :return: """ - app_process = self._start_emulation_app(emulator_settings=self._config) - with GCodeWatcher(emulator_settings=self._config) as watcher: - asyncio.run( - executable(hardware=self._emulate_hardware(settings=self._config)) - ) - yield GCodeProgram.from_g_code_watcher(watcher) - app_process.terminate() - app_process.join() + with self._emulate() as h: + with GCodeWatcher(emulator_settings=self._config) as watcher: + asyncio.run(executable(hardware=h)) + yield GCodeProgram.from_g_code_watcher(watcher) diff --git a/g-code-testing/g_code_test_data/g_code_configuration.py b/g-code-testing/g_code_test_data/g_code_configuration.py index a26f8a78a549..b80edcf092b8 100644 --- a/g-code-testing/g_code_test_data/g_code_configuration.py +++ b/g-code-testing/g_code_test_data/g_code_configuration.py @@ -84,7 +84,7 @@ class ProtocolGCodeConfirmConfig(BaseModel, SharedFunctionsMixin): name: constr(regex=r'^[a-z0-9_]*$') path: str settings: Settings - driver = 'protocol' + driver: str = 'protocol' marks: List[Mark] = [pytest.mark.g_code_confirm] class Config: @@ -99,7 +99,7 @@ class HTTPGCodeConfirmConfig(BaseModel, SharedFunctionsMixin): name: constr(regex=r'^[a-z0-9_]*$') executable: Callable settings: Settings - driver = 'http' + driver: str = 'http' marks: List[Mark] = [pytest.mark.g_code_confirm] class Config: diff --git a/g-code-testing/g_code_test_data/http/http_settings.py b/g-code-testing/g_code_test_data/http/http_settings.py index 787f265b4e08..15e5008d0d4a 100644 --- a/g-code-testing/g_code_test_data/http/http_settings.py +++ b/g-code-testing/g_code_test_data/http/http_settings.py @@ -1,9 +1,11 @@ -from opentrons.hardware_control.emulation.settings import Settings, SmoothieSettings +from opentrons.hardware_control.emulation.settings import ( + Settings, SmoothieSettings, PipetteSettings +) HTTP_SETTINGS = Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_single_v2.1", "id": "P20SV202020070101"} + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101") ) ) diff --git a/g-code-testing/g_code_test_data/http/modules/magdeck.py b/g-code-testing/g_code_test_data/http/modules/magdeck.py index 79054e5faf95..e1dc2b573dea 100644 --- a/g-code-testing/g_code_test_data/http/modules/magdeck.py +++ b/g-code-testing/g_code_test_data/http/modules/magdeck.py @@ -3,14 +3,14 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.magdeck import SERIAL as SERIAL_NUM + MAGDECK_CALIBRATE = HTTPGCodeConfirmConfig( name='magdeck_calibrate', executable=partial( post_serial_command, command=SerialCommand(command_type='calibrate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -20,7 +20,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -30,7 +30,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='engage', args=[5.1]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) diff --git a/g-code-testing/g_code_test_data/http/modules/tempdeck.py b/g-code-testing/g_code_test_data/http/modules/tempdeck.py index 6b13fce9e437..6ff1a6408386 100644 --- a/g-code-testing/g_code_test_data/http/modules/tempdeck.py +++ b/g-code-testing/g_code_test_data/http/modules/tempdeck.py @@ -3,14 +3,14 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.tempdeck import SERIAL as SERIAL_NUM + TEMPDECK_DEACTIVATE = HTTPGCodeConfirmConfig( name='tempdeck_deactivate', executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -22,7 +22,7 @@ # Keep the args at a low value because the temp starts and 0.0 and only # changes 0.25 degrees every second command=SerialCommand(command_type='set_temperature', args=[2.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -34,7 +34,7 @@ # This function does not wait on the tempdeck to finish coming to temp # so no need to set to a low value command=SerialCommand(command_type='start_set_temperature', args=[40.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) diff --git a/g-code-testing/g_code_test_data/http/modules/thermocycler.py b/g-code-testing/g_code_test_data/http/modules/thermocycler.py index 37ec7ba13bb3..1485f8719c5f 100644 --- a/g-code-testing/g_code_test_data/http/modules/thermocycler.py +++ b/g-code-testing/g_code_test_data/http/modules/thermocycler.py @@ -3,7 +3,6 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.thermocycler import SERIAL as SERIAL_NUM THERMOCYCLER_CLOSE = HTTPGCodeConfirmConfig( @@ -11,7 +10,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='close'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -21,7 +20,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='open'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -31,7 +30,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -41,7 +40,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate_block'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -51,7 +50,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate_lid'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -66,7 +65,7 @@ , 1 ] ), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -76,7 +75,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='set_lid_temperature', args=[37.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -86,7 +85,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='set_temperature', args=[1.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) diff --git a/g-code-testing/g_code_test_data/protocol/protocol_configurations.py b/g-code-testing/g_code_test_data/protocol/protocol_configurations.py index 3b889b2c1642..bd55e48dcbd2 100644 --- a/g-code-testing/g_code_test_data/protocol/protocol_configurations.py +++ b/g-code-testing/g_code_test_data/protocol/protocol_configurations.py @@ -1,4 +1,4 @@ -from opentrons.hardware_control.emulation.settings import Settings, SmoothieSettings +from opentrons.hardware_control.emulation.settings import Settings, SmoothieSettings, PipetteSettings from g_code_test_data.g_code_configuration import ProtocolGCodeConfirmConfig import pytest @@ -8,8 +8,8 @@ SWIFT_SMOOTHIE_SETTINGS = Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_multi_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_multi_v2.1", id="P20SV202020070101"), ) ) @@ -22,8 +22,8 @@ path="protocol/protocols/smoothie_protocol.py", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), ) ) ) @@ -33,8 +33,8 @@ path="protocol/protocols/2_single_channel_v2.py", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_single_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101"), ) ) ) @@ -44,8 +44,8 @@ path="protocol/protocols/2_modules_1s_1m_v2.py", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p300_single_v2.1", "id": "P20SV202020070101"}, - right={"model": "p20_multi_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101"), + right=PipetteSettings(model="p20_multi_v2.1", id="P20SV202020070101"), ) ) ) diff --git a/g-code-testing/tests/g_code_parsing/test_g_code_engine.py b/g-code-testing/tests/g_code_parsing/test_g_code_engine.py index 1a5556628bc9..e85f6b423c75 100644 --- a/g-code-testing/tests/g_code_parsing/test_g_code_engine.py +++ b/g-code-testing/tests/g_code_parsing/test_g_code_engine.py @@ -13,7 +13,6 @@ from g_code_parsing.utils import get_configuration_dir CONFIG = Settings( - host="0.0.0.0", smoothie=SmoothieSettings( left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"),