From c53bb5245a749a9e015cd4dad490e6cf1ee09419 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Tue, 12 Oct 2021 11:06:46 -0400 Subject: [PATCH 01/16] proxy. run_emulator --- .../hardware_control/emulation/app.py | 2 + .../hardware_control/emulation/magdeck.py | 5 +- .../hardware_control/emulation/proxy.py | 164 ++++++++++++++++++ .../emulation/run_emulator.py | 33 ++++ .../hardware_control/emulation/settings.py | 15 ++ 5 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 api/src/opentrons/hardware_control/emulation/proxy.py create mode 100644 api/src/opentrons/hardware_control/emulation/run_emulator.py diff --git a/api/src/opentrons/hardware_control/emulation/app.py b/api/src/opentrons/hardware_control/emulation/app.py index c0b1060f404..9cb38d3c64e 100644 --- a/api/src/opentrons/hardware_control/emulation/app.py +++ b/api/src/opentrons/hardware_control/emulation/app.py @@ -84,3 +84,5 @@ def stop(self): if __name__ == "__main__": logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) asyncio.run(ServerManager().run()) + + diff --git a/api/src/opentrons/hardware_control/emulation/magdeck.py b/api/src/opentrons/hardware_control/emulation/magdeck.py index 6ba22e465d3..bfcc8d39587 100644 --- a/api/src/opentrons/hardware_control/emulation/magdeck.py +++ b/api/src/opentrons/hardware_control/emulation/magdeck.py @@ -20,6 +20,9 @@ class MagDeckEmulator(AbstractEmulator): """Magdeck emulator""" + height: float = 0 + position: float = 0 + def __init__(self, parser: Parser) -> None: self.reset() self._parser = parser @@ -30,7 +33,7 @@ def handle(self, line: str) -> Optional[str]: joined = " ".join(r for r in results if r) return None if not joined else joined - def reset(self): + def reset(self) -> None: self.height: float = 0 self.position: float = 0 diff --git a/api/src/opentrons/hardware_control/emulation/proxy.py b/api/src/opentrons/hardware_control/emulation/proxy.py new file mode 100644 index 00000000000..8e1772d4b4b --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/proxy.py @@ -0,0 +1,164 @@ +import asyncio +import logging +from typing import Tuple, List + +from opentrons.hardware_control.emulation.settings import ProxySettings, Settings + +log = logging.getLogger(__name__) + + +class Proxy: + def __init__(self, name: str) -> None: + """Constructor.""" + self._name = name + self._cons: List[Tuple[asyncio.StreamReader, asyncio.StreamWriter]] = [] + + async def run(self, settings: ProxySettings) -> None: + """ + + Args: + settings: + + Returns: + + """ + await asyncio.gather( + self._run_emulator__server(settings), self._run_driver_server(settings) + ) + + async def _run_emulator__server(self, settings: ProxySettings) -> None: + """ + + Args: + settings: + + Returns: + + """ + log.info(f"starting {self._name} emulator server at {settings.host}:{settings.emulator_port}") + server = await asyncio.start_server(self.em_conn, settings.host, settings.emulator_port) + await server.serve_forever() + + async def _run_driver_server(self, settings: ProxySettings) -> None: + """ + + Args: + settings: + + Returns: + + """ + log.info(f"starting {self._name} driver server at {settings.host}:{settings.driver_port}") + server = await asyncio.start_server(self.driver_conn, settings.host, settings.driver_port) + await server.serve_forever() + + async def em_conn( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """ + + Args: + reader: + writer: + + Returns: + + """ + log.info(f"{self._name} emulator connected.") + self._cons.append((reader, writer,)) + + async def driver_conn( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """ + + Args: + reader: + writer: + + Returns: + + """ + log.info(f"{self._name} driver connected.") + + # Pop an emulator connection. + r, w = self._cons.pop() + + async def _read_from_driver(driver_in: asyncio.StreamReader, em_out: asyncio.StreamWriter) -> None: + while True: + d = await driver_in.read(1) + if not d: + log.info(f"{self._name} driver disconnected.") + break + em_out.write(d) + + async def _read_from_em(em_in: asyncio.StreamReader, driver_out: asyncio.StreamWriter) -> None: + while True: + d = await em_in.read(1) + if not d: + log.info(f"{self._name} emulator disconnected.") + break + driver_out.write(d) + + t1 = asyncio.get_event_loop().create_task(_read_from_driver(reader, w)) + t2 = asyncio.get_event_loop().create_task(_read_from_em(r, writer)) + await t1 + t2.cancel() + try: + await t2 + except asyncio.CancelledError: + pass + + # Return the emulator connection to the pool. + self._cons.append((r, w)) + + log.info("done") + + +async def em_server(proxy: Proxy) -> None: + """ + + Args: + proxy: + + Returns: + + """ + server = await asyncio.start_server(proxy.em_conn, "localhost", 1234) + await server.serve_forever() + + +async def driver_server(proxy: Proxy) -> None: + """ + + Args: + proxy: + + Returns: + + """ + server = await asyncio.start_server(proxy.driver_conn, "localhost", 1235) + await server.serve_forever() + + +async def run() -> None: + """ + + Returns: + + """ + settings = Settings() + + await asyncio.gather( + Proxy("smoothie_proxy").run(settings.smoothie_proxy), + Proxy("magdeck_proxy").run(settings.magdeck_proxy), + Proxy("temperature_proxy").run(settings.temperature_proxy), + Proxy("thermocycler_proxy").run(settings.thermocycler_proxy), + Proxy("heatershaker_proxy").run(settings.heatershaker_proxy), + ) + pass + + +if __name__ == "__main__": + logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) + asyncio.run(run()) diff --git a/api/src/opentrons/hardware_control/emulation/run_emulator.py b/api/src/opentrons/hardware_control/emulation/run_emulator.py new file mode 100644 index 00000000000..677077512a6 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/run_emulator.py @@ -0,0 +1,33 @@ +import asyncio + +from opentrons.hardware_control.emulation.abstract_emulator import \ + AbstractEmulator +from opentrons.hardware_control.emulation.connection_handler import \ + ConnectionHandler +from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator +from opentrons.hardware_control.emulation.parser import Parser +from opentrons.hardware_control.emulation.settings import Settings + + +async def run_emulator(host: str, port: int, emulator: AbstractEmulator) -> None: + """ + + Args: + host: + port: + emulator: + + Returns: + + """ + r, w = await asyncio.open_connection(host, port) + connection = ConnectionHandler(emulator) + await connection(r, w) + + +if __name__ == '__main__': + settings = Settings() + + e = MagDeckEmulator(Parser()) + + asyncio.run(run_emulator("localhost", settings.magdeck_proxy.emulator_port, e)) \ No newline at end of file diff --git a/api/src/opentrons/hardware_control/emulation/settings.py b/api/src/opentrons/hardware_control/emulation/settings.py index 947092e95a1..1c2cdd92406 100644 --- a/api/src/opentrons/hardware_control/emulation/settings.py +++ b/api/src/opentrons/hardware_control/emulation/settings.py @@ -1,3 +1,5 @@ +from typing_extensions import Literal + from pydantic import BaseSettings, BaseModel @@ -15,10 +17,23 @@ class SmoothieSettings(BaseModel): ) +class ProxySettings(BaseSettings): + """Settings for a proxy.""" + host: str = "0.0.0.0" + emulator_port: int + driver_port: int + + class Settings(BaseSettings): smoothie: SmoothieSettings = SmoothieSettings() host: str = "0.0.0.0" + heatershaker_proxy: ProxySettings = ProxySettings(emulator_port=9000, driver_port=9995) + smoothie_proxy: ProxySettings = ProxySettings(emulator_port=9001, driver_port=9996) + thermocycler_proxy: ProxySettings = ProxySettings(emulator_port=9002, driver_port=9997) + temperature_proxy: ProxySettings = ProxySettings(emulator_port=9003, driver_port=9998) + magdeck_proxy: ProxySettings = ProxySettings(emulator_port=9004, driver_port=9999) + class Config: env_prefix = "OT_EMULATOR_" From 48c1f79e96ce67fefc929618046650004194ad5b Mon Sep 17 00:00:00 2001 From: Derek Maggio Date: Thu, 14 Oct 2021 07:22:10 -0700 Subject: [PATCH 02/16] Linting/Formatting --- .../hardware_control/emulation/app.py | 2 - .../hardware_control/emulation/proxy.py | 39 ++++++++++++++----- .../emulation/run_emulator.py | 10 ++--- .../hardware_control/emulation/settings.py | 17 +++++--- 4 files changed, 44 insertions(+), 24 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/app.py b/api/src/opentrons/hardware_control/emulation/app.py index 9cb38d3c64e..c0b1060f404 100644 --- a/api/src/opentrons/hardware_control/emulation/app.py +++ b/api/src/opentrons/hardware_control/emulation/app.py @@ -84,5 +84,3 @@ def stop(self): if __name__ == "__main__": logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) asyncio.run(ServerManager().run()) - - diff --git a/api/src/opentrons/hardware_control/emulation/proxy.py b/api/src/opentrons/hardware_control/emulation/proxy.py index 8e1772d4b4b..ec49bc5853d 100644 --- a/api/src/opentrons/hardware_control/emulation/proxy.py +++ b/api/src/opentrons/hardware_control/emulation/proxy.py @@ -35,11 +35,16 @@ async def _run_emulator__server(self, settings: ProxySettings) -> None: Returns: """ - log.info(f"starting {self._name} emulator server at {settings.host}:{settings.emulator_port}") - server = await asyncio.start_server(self.em_conn, settings.host, settings.emulator_port) + log.info( + f"starting {self._name} emulator server at " + f"{settings.host}:{settings.emulator_port}" + ) + server = await asyncio.start_server( + self.em_conn, settings.host, settings.emulator_port + ) await server.serve_forever() - async def _run_driver_server(self, settings: ProxySettings) -> None: + async def _run_driver_server(self, settings: ProxySettings) -> None: """ Args: @@ -48,12 +53,17 @@ async def _run_driver_server(self, settings: ProxySettings) -> None: Returns: """ - log.info(f"starting {self._name} driver server at {settings.host}:{settings.driver_port}") - server = await asyncio.start_server(self.driver_conn, settings.host, settings.driver_port) + log.info( + f"starting {self._name} driver server at " + f"{settings.host}:{settings.driver_port}" + ) + server = await asyncio.start_server( + self.driver_conn, settings.host, settings.driver_port + ) await server.serve_forever() async def em_conn( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter ) -> None: """ @@ -65,10 +75,15 @@ async def em_conn( """ log.info(f"{self._name} emulator connected.") - self._cons.append((reader, writer,)) + self._cons.append( + ( + reader, + writer, + ) + ) async def driver_conn( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter ) -> None: """ @@ -84,7 +99,9 @@ async def driver_conn( # Pop an emulator connection. r, w = self._cons.pop() - async def _read_from_driver(driver_in: asyncio.StreamReader, em_out: asyncio.StreamWriter) -> None: + async def _read_from_driver( + driver_in: asyncio.StreamReader, em_out: asyncio.StreamWriter + ) -> None: while True: d = await driver_in.read(1) if not d: @@ -92,7 +109,9 @@ async def _read_from_driver(driver_in: asyncio.StreamReader, em_out: asyncio.Str break em_out.write(d) - async def _read_from_em(em_in: asyncio.StreamReader, driver_out: asyncio.StreamWriter) -> None: + async def _read_from_em( + em_in: asyncio.StreamReader, driver_out: asyncio.StreamWriter + ) -> None: while True: d = await em_in.read(1) if not d: diff --git a/api/src/opentrons/hardware_control/emulation/run_emulator.py b/api/src/opentrons/hardware_control/emulation/run_emulator.py index 677077512a6..75c0a8c8cad 100644 --- a/api/src/opentrons/hardware_control/emulation/run_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/run_emulator.py @@ -1,9 +1,7 @@ import asyncio -from opentrons.hardware_control.emulation.abstract_emulator import \ - AbstractEmulator -from opentrons.hardware_control.emulation.connection_handler import \ - ConnectionHandler +from opentrons.hardware_control.emulation.abstract_emulator import AbstractEmulator +from opentrons.hardware_control.emulation.connection_handler import ConnectionHandler from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator from opentrons.hardware_control.emulation.parser import Parser from opentrons.hardware_control.emulation.settings import Settings @@ -25,9 +23,9 @@ async def run_emulator(host: str, port: int, emulator: AbstractEmulator) -> None await connection(r, w) -if __name__ == '__main__': +if __name__ == "__main__": settings = Settings() e = MagDeckEmulator(Parser()) - asyncio.run(run_emulator("localhost", settings.magdeck_proxy.emulator_port, e)) \ No newline at end of file + asyncio.run(run_emulator("localhost", settings.magdeck_proxy.emulator_port, e)) diff --git a/api/src/opentrons/hardware_control/emulation/settings.py b/api/src/opentrons/hardware_control/emulation/settings.py index 1c2cdd92406..2afb348aa10 100644 --- a/api/src/opentrons/hardware_control/emulation/settings.py +++ b/api/src/opentrons/hardware_control/emulation/settings.py @@ -1,5 +1,3 @@ -from typing_extensions import Literal - from pydantic import BaseSettings, BaseModel @@ -19,6 +17,7 @@ class SmoothieSettings(BaseModel): class ProxySettings(BaseSettings): """Settings for a proxy.""" + host: str = "0.0.0.0" emulator_port: int driver_port: int @@ -29,11 +28,17 @@ class Settings(BaseSettings): host: str = "0.0.0.0" - heatershaker_proxy: ProxySettings = ProxySettings(emulator_port=9000, driver_port=9995) + heatershaker_proxy: ProxySettings = ProxySettings( + emulator_port=9000, driver_port=9995 + ) smoothie_proxy: ProxySettings = ProxySettings(emulator_port=9001, driver_port=9996) - thermocycler_proxy: ProxySettings = ProxySettings(emulator_port=9002, driver_port=9997) - temperature_proxy: ProxySettings = ProxySettings(emulator_port=9003, driver_port=9998) - magdeck_proxy: ProxySettings = ProxySettings(emulator_port=9004, driver_port=9999) + thermocycler_proxy: ProxySettings = ProxySettings( + emulator_port=9002, driver_port=9997 + ) + temperature_proxy: ProxySettings = ProxySettings( + emulator_port=9003, driver_port=9998 + ) + magdeck_proxy: ProxySettings = ProxySettings(emulator_port=9004, driver_port=9999) class Config: env_prefix = "OT_EMULATOR_" From 8b49b1d3d4d72234008709619a749408571363b8 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Thu, 14 Oct 2021 10:40:51 -0400 Subject: [PATCH 03/16] emulator --- .../hardware_control/emulation/app.py | 96 +++--- .../emulation/module_server.py | 113 +++++++ .../hardware_control/emulation/proxy.py | 297 ++++++++++-------- .../emulation/run_emulator.py | 29 +- .../emulation/run_module_emulator.py | 52 +++ .../hardware_control/emulation/settings.py | 12 +- .../hardware_control/emulation/smoothie.py | 41 ++- .../hardware_control/emulation/test_proxy.py | 137 ++++++++ .../hardware_control/integration/conftest.py | 4 +- 9 files changed, 558 insertions(+), 223 deletions(-) create mode 100644 api/src/opentrons/hardware_control/emulation/module_server.py create mode 100644 api/src/opentrons/hardware_control/emulation/run_module_emulator.py create mode 100644 api/tests/opentrons/hardware_control/emulation/test_proxy.py diff --git a/api/src/opentrons/hardware_control/emulation/app.py b/api/src/opentrons/hardware_control/emulation/app.py index c0b1060f404..fe09ff2076c 100644 --- a/api/src/opentrons/hardware_control/emulation/app.py +++ b/api/src/opentrons/hardware_control/emulation/app.py @@ -1,86 +1,60 @@ import asyncio import logging -from opentrons.hardware_control.emulation.connection_handler import ConnectionHandler -from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator + +from opentrons.hardware_control.emulation.module_server import ModuleStatusServer from opentrons.hardware_control.emulation.parser import Parser +from opentrons.hardware_control.emulation.proxy import Proxy +from opentrons.hardware_control.emulation.run_emulator import run_emulator_server from opentrons.hardware_control.emulation.settings import Settings -from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator -from opentrons.hardware_control.emulation.thermocycler import ThermocyclerEmulator from opentrons.hardware_control.emulation.smoothie import SmoothieEmulator logger = logging.getLogger(__name__) -SMOOTHIE_PORT = 9996 -THERMOCYCLER_PORT = 9997 -TEMPDECK_PORT = 9998 -MAGDECK_PORT = 9999 - +class Application: + """The emulator application.""" -class ServerManager: - """ - Class to start and stop emulated smoothie and modules. - """ + def __init__(self, settings: Settings) -> None: + """Constructor. - def __init__(self, settings=Settings()) -> None: - host = settings.host - self._mag_emulator = MagDeckEmulator(parser=Parser()) - self._temp_emulator = TempDeckEmulator(parser=Parser()) - self._therm_emulator = ThermocyclerEmulator(parser=Parser()) + Args: + settings: Application settings. + """ + self._settings = settings + self._status_server = ModuleStatusServer(settings.module_server) self._smoothie_emulator = SmoothieEmulator( parser=Parser(), settings=settings.smoothie ) - - self._mag_server = self._create_server( - host=host, - port=MAGDECK_PORT, - handler=ConnectionHandler(self._mag_emulator), + self._magdeck = Proxy( + "magdeck", self._status_server, self._settings.magdeck_proxy ) - self._temp_server = self._create_server( - host=host, - port=TEMPDECK_PORT, - handler=ConnectionHandler(self._temp_emulator), + self._temperature = Proxy( + "temperature", self._status_server, self._settings.temperature_proxy ) - self._therm_server = self._create_server( - host=host, - port=THERMOCYCLER_PORT, - handler=ConnectionHandler(self._therm_emulator), + self._thermocycler = Proxy( + "thermocycler", self._status_server, self._settings.thermocycler_proxy ) - self._smoothie_server = self._create_server( - host=host, - port=SMOOTHIE_PORT, - handler=ConnectionHandler(self._smoothie_emulator), + self._heatershaker = Proxy( + "heatershaker", self._status_server, self._settings.heatershaker_proxy ) - async def run(self): + async def run(self) -> None: + """Run the application.""" await asyncio.gather( - self._mag_server, - self._temp_server, - self._therm_server, - self._smoothie_server, + self._status_server.run(), + run_emulator_server( + host=self._settings.smoothie.host, + port=self._settings.smoothie.port, + emulator=self._smoothie_emulator, + ), + self._magdeck.run(), + self._temperature.run(), + self._thermocycler.run(), + self._heatershaker.run(), ) - @staticmethod - async def _create_server(host: str, port: int, handler: ConnectionHandler) -> None: - """Run a server.""" - server = await asyncio.start_server(handler, host, port) - - async with server: - await server.serve_forever() - - def reset(self): - self._smoothie_emulator.reset() - self._mag_emulator.reset() - self._temp_emulator.reset() - self._therm_emulator.reset() - - def stop(self): - self._smoothie_server.close() - self._temp_server.close() - self._therm_server.close() - self._mag_server.close() - if __name__ == "__main__": logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) - asyncio.run(ServerManager().run()) + s = Settings() + asyncio.run(Application(settings=s).run()) diff --git a/api/src/opentrons/hardware_control/emulation/module_server.py b/api/src/opentrons/hardware_control/emulation/module_server.py new file mode 100644 index 00000000000..0a6443899d9 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import asyncio +import logging +from typing_extensions import Literal +from typing import Dict, List, Set +from pydantic import BaseModel + +from opentrons.hardware_control.emulation.proxy import ProxyListener +from opentrons.hardware_control.emulation.settings import ModuleServerSettings + + +log = logging.getLogger(__name__) + + +class ModuleStatusServer(ProxyListener): + """Server notifying of module connections.""" + + def __init__(self, settings: ModuleServerSettings) -> None: + """Constructor + + Args: + settings: app settings + """ + self._settings = settings + self._connections: Dict[str, Connection] = {} + self._clients: Set[asyncio.StreamWriter] = set() + + def on_server_connected( + self, server_type: str, client_uri: str, identifier: str + ) -> None: + """Called when a new module has connected. + + Args: + server_type: the type of module + client_uri: the url string for a driver to connect to + identifier: unique id for connection + + Returns: None + + """ + log.info(f"On connected {server_type} {client_uri} {identifier}") + connection = Connection( + module_type=server_type, url=client_uri, identifier=identifier + ) + self._connections[identifier] = connection + for c in self._clients: + c.write( + Message(status="connected", connections=[connection]).json().encode() + ) + c.write(b"\n") + + def on_server_disconnected(self, identifier: str) -> None: + """Called when a module has disconnected. + + Args: + identifier: unique id for the connection + + Returns: None + """ + log.info(f"On disconnected {identifier}") + try: + connection = self._connections[identifier] + del self._connections[identifier] + for c in self._clients: + c.write( + Message(status="disconnected", connections=[connection]) + .json() + .encode() + ) + c.write(b"\n") + except KeyError: + log.exception("Failed to find identifier") + + async def run(self) -> None: + """""" + server = await asyncio.start_server( + self._handle_connection, host=self._settings.host, port=self._settings.port + ) + await server.serve_forever() + + async def _handle_connection( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """Handle a client connection to the server.""" + log.info("Client connected to module server.") + + m = Message(status="dump", connections=list(self._connections.values())) + + writer.write(m.json().encode()) + writer.write(b"\n") + + self._clients.add(writer) + + while True: + if b"" == await reader.read(): + self._clients.remove(writer) + break + + +class Connection(BaseModel): + """Model a single module connection.""" + + url: str + module_type: str + identifier: str + + +class Message(BaseModel): + """A message sent to module server clients.""" + + status: Literal["connected", "disconnected", "dump"] + connections: List[Connection] diff --git a/api/src/opentrons/hardware_control/emulation/proxy.py b/api/src/opentrons/hardware_control/emulation/proxy.py index ec49bc5853d..b921c0db3f7 100644 --- a/api/src/opentrons/hardware_control/emulation/proxy.py +++ b/api/src/opentrons/hardware_control/emulation/proxy.py @@ -1,183 +1,228 @@ +from __future__ import annotations import asyncio import logging -from typing import Tuple, List +import socket +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List -from opentrons.hardware_control.emulation.settings import ProxySettings, Settings +from opentrons.hardware_control.emulation.settings import ProxySettings log = logging.getLogger(__name__) -class Proxy: - def __init__(self, name: str) -> None: - """Constructor.""" - self._name = name - self._cons: List[Tuple[asyncio.StreamReader, asyncio.StreamWriter]] = [] +@dataclass(frozen=True) +class Connection: + """A connection.""" - async def run(self, settings: ProxySettings) -> None: - """ + identifier: str + reader: asyncio.StreamReader + writer: asyncio.StreamWriter - Args: - settings: - Returns: +class ProxyListener(ABC): + @abstractmethod + def on_server_connected( + self, server_type: str, client_uri: str, identifier: str + ) -> None: + """Called when a new server connects.""" + ... - """ - await asyncio.gather( - self._run_emulator__server(settings), self._run_driver_server(settings) - ) + @abstractmethod + def on_server_disconnected(self, identifier: str) -> None: + """Called when a server disconnects.""" + ... - async def _run_emulator__server(self, settings: ProxySettings) -> None: - """ - Args: - settings: +class Proxy: + """A class that has servers (emulators) connect on one port and clients + (drivers) on another. A server connection will be added to a collection. A + client connection will check for a server connection and if available will + have its write stream attached to the servers read stream and vice versa.""" - Returns: + def __init__( + self, name: str, listener: ProxyListener, settings: ProxySettings + ) -> None: + """Constructor. + Args: + name: Proxy name. + listener: Connection even listener. + settings: The proxy settings. """ + self._name = name + self._settings = settings + self._event_listener = listener + self._cons: List[Connection] = [] + + @property + def name(self) -> str: + """Return the name of the proxy.""" + return self._name + + async def run(self) -> None: + """Run the server.""" + await asyncio.gather( + self._listen_server_connections(), + self._listen_client_connections(), + ) + + async def _listen_server_connections(self) -> None: + """Run the server listener.""" log.info( - f"starting {self._name} emulator server at " - f"{settings.host}:{settings.emulator_port}" + f"starting {self._name} server connection server at " + f"{self._settings.host}:{self._settings.emulator_port}" ) server = await asyncio.start_server( - self.em_conn, settings.host, settings.emulator_port + self._handle_server_connection, + self._settings.host, + self._settings.emulator_port, ) await server.serve_forever() - async def _run_driver_server(self, settings: ProxySettings) -> None: - """ - - Args: - settings: - - Returns: - - """ + async def _listen_client_connections(self) -> None: + """Run the client listener.""" log.info( - f"starting {self._name} driver server at " - f"{settings.host}:{settings.driver_port}" + f"starting {self._name} client connection server at " + f"{self._settings.host}:{self._settings.driver_port}" ) server = await asyncio.start_server( - self.driver_conn, settings.host, settings.driver_port + self._handle_client_connection, + self._settings.host, + self._settings.driver_port, ) await server.serve_forever() - async def em_conn( + async def _handle_server_connection( self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter ) -> None: - """ + """Handle a server connection. + + A new connection will be added to the our connection collection. Args: - reader: - writer: + reader: Reader + writer: Writer Returns: - + None """ log.info(f"{self._name} emulator connected.") - self._cons.append( - ( - reader, - writer, - ) + connection = Connection( + identifier=str(uuid.uuid1()), reader=reader, writer=writer + ) + self._cons.append(connection) + self._event_listener.on_server_connected( + server_type=self._name, + client_uri=f"{socket.gethostname()}:{self._settings.driver_port}", + identifier=connection.identifier, ) - async def driver_conn( + async def _handle_client_connection( self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter ) -> None: - """ + """Handle a client connection. + + Attempt to match the client steam with an available server stream. Args: - reader: - writer: + reader: Reader + writer: Writer Returns: - + None """ - log.info(f"{self._name} driver connected.") - - # Pop an emulator connection. - r, w = self._cons.pop() - - async def _read_from_driver( - driver_in: asyncio.StreamReader, em_out: asyncio.StreamWriter - ) -> None: + try: while True: - d = await driver_in.read(1) - if not d: - log.info(f"{self._name} driver disconnected.") + # Pop an emulator connection. + connection = self._cons.pop(0) + if not connection.reader.at_eof(): break - em_out.write(d) + else: + log.info(f"{self._name} server connection terminated") + self._event_listener.on_server_disconnected(connection.identifier) + except IndexError: + log.info(f"{self._name} no emulator connected.") + writer.close() + return - async def _read_from_em( - em_in: asyncio.StreamReader, driver_out: asyncio.StreamWriter - ) -> None: - while True: - d = await em_in.read(1) - if not d: - log.info(f"{self._name} emulator disconnected.") - break - driver_out.write(d) + log.info( + f"{self._name} " + f"client at {writer.transport.get_extra_info('socket')}" + f" connected to {connection.writer.transport.get_extra_info('socket')}." + ) - t1 = asyncio.get_event_loop().create_task(_read_from_driver(reader, w)) - t2 = asyncio.get_event_loop().create_task(_read_from_em(r, writer)) - await t1 - t2.cancel() - try: - await t2 - except asyncio.CancelledError: - pass + await self._handle_proxy( + driver=Connection( + reader=reader, writer=writer, identifier=connection.identifier + ), + server=connection, + ) # Return the emulator connection to the pool. - self._cons.append((r, w)) - - log.info("done") - - -async def em_server(proxy: Proxy) -> None: - """ - - Args: - proxy: - - Returns: - - """ - server = await asyncio.start_server(proxy.em_conn, "localhost", 1234) - await server.serve_forever() - - -async def driver_server(proxy: Proxy) -> None: - """ + if not connection.reader.at_eof(): + log.info(f"{self._name} returning connection to pool") + self._cons.append(connection) + else: + log.info(f"{self._name} server connection terminated") + self._event_listener.on_server_disconnected(connection.identifier) - Args: - proxy: + async def _handle_proxy(self, driver: Connection, server: Connection) -> None: + """Connect the driver to the emulator. - Returns: - - """ - server = await asyncio.start_server(proxy.driver_conn, "localhost", 1235) - await server.serve_forever() - - -async def run() -> None: - """ - - Returns: + Args: + driver: Driver connection + server: Emulator connection - """ - settings = Settings() + Returns: + None + """ + loop = asyncio.get_event_loop() + read_from_client_task = loop.create_task( + self._data_router(driver, server, False) + ) + read_from_server_task = loop.create_task( + self._data_router(server, driver, True) + ) + await read_from_client_task + read_from_server_task.cancel() + try: + await read_from_server_task + except asyncio.CancelledError: + log.exception("Server task cancelled") + pass - await asyncio.gather( - Proxy("smoothie_proxy").run(settings.smoothie_proxy), - Proxy("magdeck_proxy").run(settings.magdeck_proxy), - Proxy("temperature_proxy").run(settings.temperature_proxy), - Proxy("thermocycler_proxy").run(settings.thermocycler_proxy), - Proxy("heatershaker_proxy").run(settings.heatershaker_proxy), - ) - pass + @staticmethod + async def _data_router( + in_connection: Connection, + out_connection: Connection, + close_other_on_disconnect: bool, + ) -> None: + """Route date from in to out. + Args: + in_connection: connection to read from. + out_connection: connection to write to + close_other_on_disconnect: whether the other connection should + be closed if the in_connection is closed. + A driver disconnect should close connection with emulator, while + an emulator disconnect should close the attached driver. -if __name__ == "__main__": - logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) - asyncio.run(run()) + Returns: + None + """ + while True: + try: + d = await in_connection.reader.read(1) + if not d: + log.info( + f"{in_connection.writer.transport.get_extra_info('socket')} disconnected." + ) + break + out_connection.writer.write(d) + except ConnectionError: + log.exception(f"connection error in data router") + break + if close_other_on_disconnect: + out_connection.writer.close() diff --git a/api/src/opentrons/hardware_control/emulation/run_emulator.py b/api/src/opentrons/hardware_control/emulation/run_emulator.py index 75c0a8c8cad..2ed1a21a696 100644 --- a/api/src/opentrons/hardware_control/emulation/run_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/run_emulator.py @@ -1,31 +1,22 @@ import asyncio +import logging from opentrons.hardware_control.emulation.abstract_emulator import AbstractEmulator from opentrons.hardware_control.emulation.connection_handler import ConnectionHandler -from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator -from opentrons.hardware_control.emulation.parser import Parser -from opentrons.hardware_control.emulation.settings import Settings +log = logging.getLogger(__name__) -async def run_emulator(host: str, port: int, emulator: AbstractEmulator) -> None: - """ - Args: - host: - port: - emulator: - - Returns: - - """ +async def run_emulator_client(host: str, port: int, emulator: AbstractEmulator) -> None: + """Run an emulator as a client.""" + log.info(f"Connecting to {emulator.__class__.__name__} at {host}:{port}") r, w = await asyncio.open_connection(host, port) connection = ConnectionHandler(emulator) await connection(r, w) -if __name__ == "__main__": - settings = Settings() - - e = MagDeckEmulator(Parser()) - - asyncio.run(run_emulator("localhost", settings.magdeck_proxy.emulator_port, e)) +async def run_emulator_server(host: str, port: int, emulator: AbstractEmulator) -> None: + """Run an emulator as a server.""" + log.info(f"Starting {emulator.__class__.__name__} at {host}:{port}") + server = await asyncio.start_server(ConnectionHandler(emulator), host, port) + await server.serve_forever() diff --git a/api/src/opentrons/hardware_control/emulation/run_module_emulator.py b/api/src/opentrons/hardware_control/emulation/run_module_emulator.py new file mode 100644 index 00000000000..51fa5c8bfc8 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/run_module_emulator.py @@ -0,0 +1,52 @@ +import logging +import asyncio +from argparse import ArgumentParser +from typing import Dict, Callable + +from opentrons.hardware_control.emulation.abstract_emulator import AbstractEmulator +from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator +from opentrons.hardware_control.emulation.parser import Parser +from opentrons.hardware_control.emulation.run_emulator import run_emulator_client +from opentrons.hardware_control.emulation.settings import Settings, ProxySettings +from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator +from opentrons.hardware_control.emulation.thermocycler import ThermocyclerEmulator + +emulator_builder: Dict[str, Callable[[Settings], AbstractEmulator]] = { + "magdeck": lambda s: MagDeckEmulator(Parser()), + "temperature": lambda s: TempDeckEmulator(Parser()), + "thermocycler": lambda s: ThermocyclerEmulator(Parser()), +} + +emulator_port: Dict[str, Callable[[Settings], ProxySettings]] = { + "magdeck": lambda s: s.magdeck_proxy, + "temperature": lambda s: s.temperature_proxy, + "thermocycler": lambda s: s.thermocycler_proxy, +} + + +def run(emulator_name: str, host: str) -> None: + """Run an emulator. + + Args: + emulator_name: Name of emulator. This must be a key in emulator_builder + host: host to connect to. + + Returns: + None + """ + settings = Settings() + + e = emulator_builder[emulator_name](settings) + proxy_settings = emulator_port[emulator_name](settings) + + asyncio.run(run_emulator_client(host, proxy_settings.emulator_port, e)) + + +if __name__ == "__main__": + a = ArgumentParser() + a.add_argument("emulator", type=str, choices=emulator_builder.keys()) + a.add_argument("host", type=str) + args = a.parse_args() + + logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) + run(args.emulator, args.host) diff --git a/api/src/opentrons/hardware_control/emulation/settings.py b/api/src/opentrons/hardware_control/emulation/settings.py index 2afb348aa10..8e62cfc1192 100644 --- a/api/src/opentrons/hardware_control/emulation/settings.py +++ b/api/src/opentrons/hardware_control/emulation/settings.py @@ -13,6 +13,8 @@ class SmoothieSettings(BaseModel): right: PipetteSettings = PipetteSettings( model="p20_single_v2.0", id="P20SV202020070101" ) + host: str = "0.0.0.0" + port: int = 9996 class ProxySettings(BaseSettings): @@ -23,6 +25,13 @@ class ProxySettings(BaseSettings): driver_port: int +class ModuleServerSettings(BaseModel): + """Settings for the module server""" + + host: str = "0.0.0.0" + port: int = 8888 + + class Settings(BaseSettings): smoothie: SmoothieSettings = SmoothieSettings() @@ -31,7 +40,6 @@ class Settings(BaseSettings): heatershaker_proxy: ProxySettings = ProxySettings( emulator_port=9000, driver_port=9995 ) - smoothie_proxy: ProxySettings = ProxySettings(emulator_port=9001, driver_port=9996) thermocycler_proxy: ProxySettings = ProxySettings( emulator_port=9002, driver_port=9997 ) @@ -42,3 +50,5 @@ class Settings(BaseSettings): class Config: env_prefix = "OT_EMULATOR_" + + module_server: ModuleServerSettings = ModuleServerSettings() diff --git a/api/src/opentrons/hardware_control/emulation/smoothie.py b/api/src/opentrons/hardware_control/emulation/smoothie.py index 435886f0128..c06960724c2 100644 --- a/api/src/opentrons/hardware_control/emulation/smoothie.py +++ b/api/src/opentrons/hardware_control/emulation/smoothie.py @@ -6,6 +6,7 @@ import logging import re from typing import Optional, Dict +from typing_extensions import Literal from opentrons import _find_smoothie_file from opentrons.drivers import utils @@ -24,9 +25,33 @@ class SmoothieEmulator(AbstractEmulator): WRITE_INSTRUMENT_RE = re.compile(r"(?P[LR])\s*(?P[a-f0-9]+)") INSTRUMENT_AND_MODEL_STRING_LENGTH = 64 + _version_string: str + _pos: Dict[str, float] + _home_status: Dict[str, bool] + _speed: float + _pipette_model: Dict[Literal["L", "R"], str] + _pipette_id: Dict[Literal["L", "R"], str] + def __init__(self, parser: Parser, settings: SmoothieSettings) -> None: + """Constructor. + + Args: + parser: GCODE Parser. + settings: emulator settings. + """ self._parser = parser self._settings = settings + self._gcode_to_function_mapping = { + GCODE.HOMING_STATUS.value: self._get_homing_status, + GCODE.CURRENT_POSITION.value: self._get_current_position, + GCODE.VERSION.value: self._get_version, + GCODE.READ_INSTRUMENT_ID.value: self._get_pipette_id, + GCODE.READ_INSTRUMENT_MODEL.value: self._get_pipette_model, + GCODE.WRITE_INSTRUMENT_ID.value: self._set_pipette_id, + GCODE.WRITE_INSTRUMENT_MODEL.value: self._set_pipette_model, + GCODE.MOVE.value: self._move_gantry, + GCODE.HOME.value: self._home_gantry, + } self.reset() def handle(self, line: str) -> Optional[str]: @@ -35,7 +60,7 @@ def handle(self, line: str) -> Optional[str]: joined = " ".join(r for r in results if r) return None if not joined else joined - def reset(self): + def reset(self) -> None: _, fw_version = _find_smoothie_file() self._version_string = ( f"Build version: {fw_version}, Build date: CURRENT, " @@ -71,19 +96,7 @@ def reset(self): ), } - self._gcode_to_function_mapping = { - GCODE.HOMING_STATUS.value: self._get_homing_status, - GCODE.CURRENT_POSITION.value: self._get_current_position, - GCODE.VERSION.value: self._get_version, - GCODE.READ_INSTRUMENT_ID.value: self._get_pipette_id, - GCODE.READ_INSTRUMENT_MODEL.value: self._get_pipette_model, - GCODE.WRITE_INSTRUMENT_ID.value: self._set_pipette_id, - GCODE.WRITE_INSTRUMENT_MODEL.value: self._set_pipette_model, - GCODE.MOVE.value: self._move_gantry, - GCODE.HOME.value: self._home_gantry, - } - - def get_current_position(self): + def get_current_position(self) -> Dict[str, float]: return self._pos def _get_homing_status(self, command: Command) -> str: diff --git a/api/tests/opentrons/hardware_control/emulation/test_proxy.py b/api/tests/opentrons/hardware_control/emulation/test_proxy.py new file mode 100644 index 00000000000..98442576436 --- /dev/null +++ b/api/tests/opentrons/hardware_control/emulation/test_proxy.py @@ -0,0 +1,137 @@ +import asyncio +from typing import AsyncIterator + +import pytest + +from opentrons.hardware_control.emulation.proxy import Proxy, ProxySettings + + +@pytest.fixture +def setting() -> ProxySettings: + """Proxy settings fixture.""" + return ProxySettings(emulator_port=12345, driver_port=12346) + + +@pytest.fixture +async def subject( + loop: asyncio.AbstractEventLoop, setting: ProxySettings +) -> AsyncIterator[Proxy]: + """Test subject.""" + p = Proxy("proxy") + task = loop.create_task(p.run(setting)) + yield p + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + +async def test_driver_route_message(subject: Proxy, setting: ProxySettings) -> None: + """It should route a message to an emulator.""" + emulator = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) + driver[1].write(b"abc") + r = await emulator[0].read(3) + assert r == b"abc" + + +async def test_emulator_route_message(subject: Proxy, setting: ProxySettings) -> None: + """It should route a message to a driver.""" + emulator = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) + emulator[1].write(b"abc") + r = await driver[0].read(3) + assert r == b"abc" + + +async def test_driver_route_message_two_connections( + subject: Proxy, setting: ProxySettings +) -> None: + """It should route messages to correct emulator.""" + emulator1 = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + emulator2 = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + driver1 = await asyncio.open_connection(host="localhost", port=setting.driver_port) + driver2 = await asyncio.open_connection(host="localhost", port=setting.driver_port) + driver1[1].write(b"abc") + driver2[1].write(b"cba") + r1 = await emulator1[0].read(3) + r2 = await emulator2[0].read(3) + assert r1 == b"abc" + assert r2 == b"cba" + + +async def test_emulator_route_message_two_connections( + subject: Proxy, setting: ProxySettings +) -> None: + """It should route messages to correct driver.""" + emulator1 = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + emulator2 = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + driver1 = await asyncio.open_connection(host="localhost", port=setting.driver_port) + driver2 = await asyncio.open_connection(host="localhost", port=setting.driver_port) + emulator1[1].write(b"abc") + emulator2[1].write(b"cba") + r1 = await driver1[0].read(3) + r2 = await driver2[0].read(3) + assert r1 == b"abc" + assert r2 == b"cba" + + +async def test_driver_and_no_emulator(subject: Proxy, setting: ProxySettings) -> None: + """It should fail to read if no emulator.""" + driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) + assert b"" == await driver[0].read(n=1) + + +async def test_two_driver_and_one_emulator( + subject: Proxy, setting: ProxySettings +) -> None: + """It should fail to read if no emulator.""" + emulator = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + driver1 = await asyncio.open_connection(host="localhost", port=setting.driver_port) + driver2 = await asyncio.open_connection(host="localhost", port=setting.driver_port) + emulator[1].write(b"abc") + assert b"abc" == await driver1[0].read(n=3) + assert b"" == await driver2[0].read(n=3) + + +async def test_driver_reconnect(subject: Proxy, setting: ProxySettings) -> None: + """It should allow a second driver to claim a formerly used emulator.""" + emulator = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) + emulator[1].write(b"abc") + assert b"abc" == await driver[0].read(n=3) + + driver[1].close() + + driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) + emulator[1].write(b"abc") + assert b"abc" == await driver[0].read(n=3) + + +async def test_emulator_disconnects(subject: Proxy, setting: ProxySettings) -> None: + """It should disconnect driver when emulator disconnects.""" + emulator = await asyncio.open_connection( + host="localhost", port=setting.emulator_port + ) + driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) + emulator[1].close() + + driver[1].write(b"123") + assert b"" == await driver[0].read(n=3) diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 2ec0ab9c45c..1a3b90fcac8 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -1,7 +1,7 @@ import pytest import threading import asyncio -from opentrons.hardware_control.emulation.app import ServerManager +from opentrons.hardware_control.emulation.app import Application from opentrons.hardware_control.emulation.settings import ( Settings, SmoothieSettings, @@ -22,7 +22,7 @@ def emulation_app(): """Run the emulators""" def runit(): - asyncio.run(ServerManager().run()) + asyncio.run(Application().run()) # TODO 20210219 # The emulators must be run in a separate thread because our serial From 27cdce0f7142c9e86d585ecea0a67aa90deb065e Mon Sep 17 00:00:00 2001 From: amit lissack Date: Tue, 19 Oct 2021 10:28:03 -0400 Subject: [PATCH 04/16] integration tests pass. --- .../hardware_control/modules/magdeck.py | 3 + .../hardware_control/modules/tempdeck.py | 1 + .../hardware_control/modules/thermocycler.py | 1 + .../hardware_control/emulation/test_proxy.py | 6 +- .../hardware_control/integration/conftest.py | 61 +++++++++++++++---- .../integration/test_controller.py | 7 ++- .../integration/test_magdeck.py | 7 ++- .../integration/test_smoothie.py | 7 ++- .../integration/test_tempdeck.py | 21 ++++--- .../integration/test_thermocycler.py | 8 +-- 10 files changed, 88 insertions(+), 34 deletions(-) diff --git a/api/src/opentrons/hardware_control/modules/magdeck.py b/api/src/opentrons/hardware_control/modules/magdeck.py index 7793ac608da..d5f7ab34884 100644 --- a/api/src/opentrons/hardware_control/modules/magdeck.py +++ b/api/src/opentrons/hardware_control/modules/magdeck.py @@ -69,6 +69,9 @@ def __init__( self._driver = driver self._current_height = 0.0 + async def cleanup(self) -> None: + await self._driver.disconnect() + @classmethod def name(cls) -> str: """Get the module name.""" diff --git a/api/src/opentrons/hardware_control/modules/tempdeck.py b/api/src/opentrons/hardware_control/modules/tempdeck.py index 0dd03ebf2bc..193758fb333 100644 --- a/api/src/opentrons/hardware_control/modules/tempdeck.py +++ b/api/src/opentrons/hardware_control/modules/tempdeck.py @@ -103,6 +103,7 @@ def __init__( async def cleanup(self) -> None: """Stop the poller task.""" await self._poller.stop_and_wait() + await self._driver.disconnect() @classmethod def name(cls) -> str: diff --git a/api/src/opentrons/hardware_control/modules/thermocycler.py b/api/src/opentrons/hardware_control/modules/thermocycler.py index a993cdc2e90..928cb2911b6 100644 --- a/api/src/opentrons/hardware_control/modules/thermocycler.py +++ b/api/src/opentrons/hardware_control/modules/thermocycler.py @@ -130,6 +130,7 @@ def __init__( async def cleanup(self) -> None: """Stop the poller task.""" await self._poller.stop_and_wait() + await self._driver.disconnect() @classmethod def name(cls) -> str: diff --git a/api/tests/opentrons/hardware_control/emulation/test_proxy.py b/api/tests/opentrons/hardware_control/emulation/test_proxy.py index 98442576436..bc128f00e86 100644 --- a/api/tests/opentrons/hardware_control/emulation/test_proxy.py +++ b/api/tests/opentrons/hardware_control/emulation/test_proxy.py @@ -2,6 +2,7 @@ from typing import AsyncIterator import pytest +from mock import MagicMock from opentrons.hardware_control.emulation.proxy import Proxy, ProxySettings @@ -17,8 +18,9 @@ async def subject( loop: asyncio.AbstractEventLoop, setting: ProxySettings ) -> AsyncIterator[Proxy]: """Test subject.""" - p = Proxy("proxy") - task = loop.create_task(p.run(setting)) + mock_listener = MagicMock() + p = Proxy("proxy", mock_listener, setting) + task = loop.create_task(p.run()) yield p task.cancel() try: diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 1a3b90fcac8..fb0b1fd76ab 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -1,28 +1,65 @@ +from time import sleep +from typing import Iterator + import pytest import threading import asyncio from opentrons.hardware_control.emulation.app import Application +from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator +from opentrons.hardware_control.emulation.parser import Parser +from opentrons.hardware_control.emulation.run_emulator import run_emulator_client from opentrons.hardware_control.emulation.settings import ( Settings, SmoothieSettings, PipetteSettings, ) +from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator +from opentrons.hardware_control.emulation.thermocycler import \ + ThermocyclerEmulator -CONFIG = Settings( - host="0.0.0.0", - smoothie=SmoothieSettings( - left=PipetteSettings(model="p20_multi_v2.0", id="P3HMV202020041605"), - right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), - ), -) + +@pytest.fixture(scope="session") +def emulator_settings() -> Settings: + """Emulator settings""" + return Settings( + host="0.0.0.0", + smoothie=SmoothieSettings( + left=PipetteSettings(model="p20_multi_v2.0", id="P3HMV202020041605"), + right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + ), + ) @pytest.fixture(scope="session") -def emulation_app(): +def emulation_app(emulator_settings: Settings) -> Iterator[None]: """Run the emulators""" - def runit(): - asyncio.run(Application().run()) + async def _() -> None: + await asyncio.gather( + # Start application + Application(settings=emulator_settings).run(), + # Add magdeck emulator + run_emulator_client( + host="localhost", + port=emulator_settings.magdeck_proxy.emulator_port, + emulator=MagDeckEmulator(Parser()), + ), + # Add temperature emulator + run_emulator_client( + host="localhost", + port=emulator_settings.temperature_proxy.emulator_port, + emulator=TempDeckEmulator(Parser()), + ), + # Add thermocycler emulator + run_emulator_client( + host="localhost", + port=emulator_settings.thermocycler_proxy.emulator_port, + emulator=ThermocyclerEmulator(Parser()), + ), + ) + + def runit() -> None: + asyncio.run(_()) # TODO 20210219 # The emulators must be run in a separate thread because our serial @@ -31,4 +68,6 @@ def runit(): t = threading.Thread(target=runit) t.daemon = True t.start() - yield t + # Give it a bit to get going. + sleep(0.5) + yield diff --git a/api/tests/opentrons/hardware_control/integration/test_controller.py b/api/tests/opentrons/hardware_control/integration/test_controller.py index 4d74b9499c5..d2a4fb35586 100644 --- a/api/tests/opentrons/hardware_control/integration/test_controller.py +++ b/api/tests/opentrons/hardware_control/integration/test_controller.py @@ -1,17 +1,18 @@ import asyncio +from typing import Iterator import pytest from opentrons import _find_smoothie_file from opentrons.config.robot_configs import build_config from opentrons.hardware_control import Controller -from opentrons.hardware_control.emulation.app import SMOOTHIE_PORT +from opentrons.hardware_control.emulation.settings import Settings from opentrons.types import Mount @pytest.fixture -async def subject(loop: asyncio.BaseEventLoop, emulation_app) -> Controller: +async def subject(loop: asyncio.BaseEventLoop, emulation_app: Iterator[None], emulator_settings: Settings) -> Controller: conf = build_config({}) - port = f"socket://127.0.0.1:{SMOOTHIE_PORT}" + port = f"socket://127.0.0.1:{emulator_settings.smoothie.port}" hc = await Controller.build(config=conf) await hc.connect(port=port) yield hc diff --git a/api/tests/opentrons/hardware_control/integration/test_magdeck.py b/api/tests/opentrons/hardware_control/integration/test_magdeck.py index dc5450fda7c..9d1f800810b 100644 --- a/api/tests/opentrons/hardware_control/integration/test_magdeck.py +++ b/api/tests/opentrons/hardware_control/integration/test_magdeck.py @@ -1,16 +1,17 @@ import asyncio +from typing import Iterator import pytest from mock import AsyncMock from opentrons.drivers.rpi_drivers.types import USBPort -from opentrons.hardware_control.emulation.app import MAGDECK_PORT +from opentrons.hardware_control.emulation.settings import Settings from opentrons.hardware_control.modules import MagDeck @pytest.fixture -async def magdeck(loop: asyncio.BaseEventLoop, emulation_app) -> MagDeck: +async def magdeck(loop: asyncio.BaseEventLoop, emulation_app: Iterator[None], emulator_settings: Settings) -> MagDeck: module = await MagDeck.build( - port=f"socket://127.0.0.1:{MAGDECK_PORT}", + port=f"socket://127.0.0.1:{emulator_settings.magdeck_proxy.driver_port}", execution_manager=AsyncMock(), usb_port=USBPort(name="", port_number=1, sub_names=[], device_path="", hub=1), loop=loop, diff --git a/api/tests/opentrons/hardware_control/integration/test_smoothie.py b/api/tests/opentrons/hardware_control/integration/test_smoothie.py index 9472e758ce3..c6e845820c6 100755 --- a/api/tests/opentrons/hardware_control/integration/test_smoothie.py +++ b/api/tests/opentrons/hardware_control/integration/test_smoothie.py @@ -1,7 +1,8 @@ import pytest +from typing import Iterator from mock import MagicMock from opentrons.drivers.types import MoveSplit -from opentrons.hardware_control.emulation.app import SMOOTHIE_PORT +from opentrons.hardware_control.emulation.settings import Settings from opentrons.config.robot_configs import ( DEFAULT_GANTRY_STEPS_PER_MM, @@ -12,10 +13,10 @@ @pytest.fixture -async def subject(emulation_app) -> SmoothieDriver: +async def subject(emulation_app: Iterator[None], emulator_settings: Settings) -> SmoothieDriver: """Smoothie driver connected to emulator.""" d = await SmoothieDriver.build( - port=f"socket://127.0.0.1:{SMOOTHIE_PORT}", config=build_config({}) + port=f"socket://127.0.0.1:{emulator_settings.smoothie.port}", config=build_config({}) ) yield d await d.disconnect() diff --git a/api/tests/opentrons/hardware_control/integration/test_tempdeck.py b/api/tests/opentrons/hardware_control/integration/test_tempdeck.py index 9026ef99829..d86aa841502 100644 --- a/api/tests/opentrons/hardware_control/integration/test_tempdeck.py +++ b/api/tests/opentrons/hardware_control/integration/test_tempdeck.py @@ -1,17 +1,22 @@ import asyncio +from typing import Iterator import pytest from opentrons.drivers.rpi_drivers.types import USBPort from opentrons.hardware_control import ExecutionManager -from opentrons.hardware_control.emulation.app import TEMPDECK_PORT +from opentrons.hardware_control.emulation.settings import Settings from opentrons.hardware_control.modules import TempDeck @pytest.fixture -async def tempdeck(loop: asyncio.BaseEventLoop, emulation_app) -> TempDeck: +async def tempdeck( + loop: asyncio.BaseEventLoop, + emulator_settings: Settings, + emulation_app: Iterator[None], +) -> TempDeck: execution_manager = ExecutionManager(loop) module = await TempDeck.build( - port=f"socket://127.0.0.1:{TEMPDECK_PORT}", + port=f"socket://127.0.0.1:{emulator_settings.temperature_proxy.driver_port}", execution_manager=execution_manager, usb_port=USBPort(name="", port_number=1, sub_names=[], device_path="", hub=1), loop=loop, @@ -22,7 +27,7 @@ async def tempdeck(loop: asyncio.BaseEventLoop, emulation_app) -> TempDeck: await module.cleanup() -def test_device_info(tempdeck) -> None: +def test_device_info(tempdeck: TempDeck) -> None: """It should have the device info.""" assert { "model": "temp_deck_v20", @@ -31,7 +36,7 @@ def test_device_info(tempdeck) -> None: } == tempdeck.device_info -async def test_set_temperature(tempdeck) -> None: +async def test_set_temperature(tempdeck: TempDeck) -> None: """It should set the temperature and return when target is reached.""" await tempdeck.wait_next_poll() assert tempdeck.live_data == { @@ -46,7 +51,7 @@ async def test_set_temperature(tempdeck) -> None: } -async def test_start_set_temperature_cool(tempdeck) -> None: +async def test_start_set_temperature_cool(tempdeck: TempDeck) -> None: """It should set the temperature and return and wait for temperature.""" await tempdeck.wait_next_poll() current = tempdeck.temperature @@ -68,7 +73,7 @@ async def test_start_set_temperature_cool(tempdeck) -> None: } -async def test_start_set_temperature_heat(tempdeck) -> None: +async def test_start_set_temperature_heat(tempdeck: TempDeck) -> None: """It should set the temperature and return and wait for temperature.""" await tempdeck.wait_next_poll() current = tempdeck.temperature @@ -90,7 +95,7 @@ async def test_start_set_temperature_heat(tempdeck) -> None: } -async def test_deactivate(tempdeck) -> None: +async def test_deactivate(tempdeck: TempDeck) -> None: """It should deactivate and move to room temperature""" await tempdeck.deactivate() await tempdeck.wait_next_poll() diff --git a/api/tests/opentrons/hardware_control/integration/test_thermocycler.py b/api/tests/opentrons/hardware_control/integration/test_thermocycler.py index be828622d34..adc88d08517 100644 --- a/api/tests/opentrons/hardware_control/integration/test_thermocycler.py +++ b/api/tests/opentrons/hardware_control/integration/test_thermocycler.py @@ -1,18 +1,18 @@ import asyncio - +from typing import Iterator import pytest from opentrons.drivers.rpi_drivers.types import USBPort from opentrons.hardware_control import ExecutionManager -from opentrons.hardware_control.emulation.app import THERMOCYCLER_PORT +from opentrons.hardware_control.emulation.settings import Settings from opentrons.hardware_control.modules import Thermocycler @pytest.fixture -async def thermocycler(loop: asyncio.BaseEventLoop, emulation_app) -> Thermocycler: +async def thermocycler(loop: asyncio.BaseEventLoop, emulation_app: Iterator[None], emulator_settings: Settings) -> Thermocycler: """Thermocycler fixture.""" execution_manager = ExecutionManager(loop) module = await Thermocycler.build( - port=f"socket://127.0.0.1:{THERMOCYCLER_PORT}", + port=f"socket://127.0.0.1:{emulator_settings.thermocycler_proxy.driver_port}", execution_manager=execution_manager, usb_port=USBPort(name="", port_number=1, sub_names=[], device_path="", hub=1), loop=loop, From 0fa09ec1b7671d1ac57e403441d160f42e3bd610 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 20 Oct 2021 10:37:11 -0400 Subject: [PATCH 05/16] lint passes. --- api/src/opentrons/hardware_control/emulation/proxy.py | 5 +++-- api/src/opentrons/hardware_control/emulation/smoothie.py | 5 ++--- .../opentrons/hardware_control/integration/conftest.py | 3 +-- .../hardware_control/integration/test_controller.py | 6 +++++- .../opentrons/hardware_control/integration/test_magdeck.py | 6 +++++- .../hardware_control/integration/test_smoothie.py | 7 +++++-- .../hardware_control/integration/test_thermocycler.py | 6 +++++- 7 files changed, 26 insertions(+), 12 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/proxy.py b/api/src/opentrons/hardware_control/emulation/proxy.py index b921c0db3f7..6877c89ebf5 100644 --- a/api/src/opentrons/hardware_control/emulation/proxy.py +++ b/api/src/opentrons/hardware_control/emulation/proxy.py @@ -217,12 +217,13 @@ async def _data_router( d = await in_connection.reader.read(1) if not d: log.info( - f"{in_connection.writer.transport.get_extra_info('socket')} disconnected." + f"{in_connection.writer.transport.get_extra_info('socket')} " + f"disconnected." ) break out_connection.writer.write(d) except ConnectionError: - log.exception(f"connection error in data router") + log.exception("connection error in data router") break if close_other_on_disconnect: out_connection.writer.close() diff --git a/api/src/opentrons/hardware_control/emulation/smoothie.py b/api/src/opentrons/hardware_control/emulation/smoothie.py index c06960724c2..d9b828011e1 100644 --- a/api/src/opentrons/hardware_control/emulation/smoothie.py +++ b/api/src/opentrons/hardware_control/emulation/smoothie.py @@ -6,7 +6,6 @@ import logging import re from typing import Optional, Dict -from typing_extensions import Literal from opentrons import _find_smoothie_file from opentrons.drivers import utils @@ -29,8 +28,8 @@ class SmoothieEmulator(AbstractEmulator): _pos: Dict[str, float] _home_status: Dict[str, bool] _speed: float - _pipette_model: Dict[Literal["L", "R"], str] - _pipette_id: Dict[Literal["L", "R"], str] + _pipette_model: Dict[str, str] + _pipette_id: Dict[str, str] def __init__(self, parser: Parser, settings: SmoothieSettings) -> None: """Constructor. diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index fb0b1fd76ab..4fb2d4d69ac 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -14,8 +14,7 @@ PipetteSettings, ) from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator -from opentrons.hardware_control.emulation.thermocycler import \ - ThermocyclerEmulator +from opentrons.hardware_control.emulation.thermocycler import ThermocyclerEmulator @pytest.fixture(scope="session") diff --git a/api/tests/opentrons/hardware_control/integration/test_controller.py b/api/tests/opentrons/hardware_control/integration/test_controller.py index d2a4fb35586..0a012eff30f 100644 --- a/api/tests/opentrons/hardware_control/integration/test_controller.py +++ b/api/tests/opentrons/hardware_control/integration/test_controller.py @@ -10,7 +10,11 @@ @pytest.fixture -async def subject(loop: asyncio.BaseEventLoop, emulation_app: Iterator[None], emulator_settings: Settings) -> Controller: +async def subject( + loop: asyncio.BaseEventLoop, + emulation_app: Iterator[None], + emulator_settings: Settings, +) -> Controller: conf = build_config({}) port = f"socket://127.0.0.1:{emulator_settings.smoothie.port}" hc = await Controller.build(config=conf) diff --git a/api/tests/opentrons/hardware_control/integration/test_magdeck.py b/api/tests/opentrons/hardware_control/integration/test_magdeck.py index 9d1f800810b..cb693650232 100644 --- a/api/tests/opentrons/hardware_control/integration/test_magdeck.py +++ b/api/tests/opentrons/hardware_control/integration/test_magdeck.py @@ -9,7 +9,11 @@ @pytest.fixture -async def magdeck(loop: asyncio.BaseEventLoop, emulation_app: Iterator[None], emulator_settings: Settings) -> MagDeck: +async def magdeck( + loop: asyncio.BaseEventLoop, + emulation_app: Iterator[None], + emulator_settings: Settings, +) -> MagDeck: module = await MagDeck.build( port=f"socket://127.0.0.1:{emulator_settings.magdeck_proxy.driver_port}", execution_manager=AsyncMock(), diff --git a/api/tests/opentrons/hardware_control/integration/test_smoothie.py b/api/tests/opentrons/hardware_control/integration/test_smoothie.py index c6e845820c6..baa75ad10e8 100755 --- a/api/tests/opentrons/hardware_control/integration/test_smoothie.py +++ b/api/tests/opentrons/hardware_control/integration/test_smoothie.py @@ -13,10 +13,13 @@ @pytest.fixture -async def subject(emulation_app: Iterator[None], emulator_settings: Settings) -> SmoothieDriver: +async def subject( + emulation_app: Iterator[None], emulator_settings: Settings +) -> SmoothieDriver: """Smoothie driver connected to emulator.""" d = await SmoothieDriver.build( - port=f"socket://127.0.0.1:{emulator_settings.smoothie.port}", config=build_config({}) + port=f"socket://127.0.0.1:{emulator_settings.smoothie.port}", + config=build_config({}), ) yield d await d.disconnect() diff --git a/api/tests/opentrons/hardware_control/integration/test_thermocycler.py b/api/tests/opentrons/hardware_control/integration/test_thermocycler.py index adc88d08517..0cb58709ead 100644 --- a/api/tests/opentrons/hardware_control/integration/test_thermocycler.py +++ b/api/tests/opentrons/hardware_control/integration/test_thermocycler.py @@ -8,7 +8,11 @@ @pytest.fixture -async def thermocycler(loop: asyncio.BaseEventLoop, emulation_app: Iterator[None], emulator_settings: Settings) -> Thermocycler: +async def thermocycler( + loop: asyncio.BaseEventLoop, + emulation_app: Iterator[None], + emulator_settings: Settings, +) -> Thermocycler: """Thermocycler fixture.""" execution_manager = ExecutionManager(loop) module = await Thermocycler.build( From b54b5137b6982450996f88e3c69d78a5f123a12d Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 20 Oct 2021 11:21:37 -0400 Subject: [PATCH 06/16] gcode parsing. --- .../hardware_control/integration/conftest.py | 4 +- .../g_code_parsing/g_code_engine.py | 80 ++++++++++++------- .../g_code_parsing/g_code_watcher.py | 28 +++---- .../g_code_parsing/test_g_code_program.py | 9 ++- 4 files changed, 71 insertions(+), 50 deletions(-) diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 4fb2d4d69ac..91cac71aca0 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -33,7 +33,7 @@ def emulator_settings() -> Settings: def emulation_app(emulator_settings: Settings) -> Iterator[None]: """Run the emulators""" - async def _() -> None: + async def _run_emulation_environment() -> None: await asyncio.gather( # Start application Application(settings=emulator_settings).run(), @@ -58,7 +58,7 @@ async def _() -> None: ) def runit() -> None: - asyncio.run(_()) + asyncio.run(_run_emulation_environment()) # TODO 20210219 # The emulators must be run in a separate thread because our serial diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index 10ad7218ad9..a6bd6f778ba 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -5,20 +5,21 @@ from typing import Generator, Callable from collections import namedtuple +from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator +from opentrons.hardware_control.emulation.parser import Parser +from opentrons.hardware_control.emulation.run_emulator import \ + run_emulator_client from opentrons.hardware_control.emulation.settings import Settings +from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator +from opentrons.hardware_control.emulation.thermocycler import \ + ThermocyclerEmulator from opentrons.protocols.parse import parse from opentrons.protocols.execution import execute from contextlib import contextmanager from opentrons.protocol_api import ProtocolContext from opentrons.config.robot_configs import build_config -from opentrons.hardware_control.emulation.app import ServerManager +from opentrons.hardware_control.emulation.app import Application from opentrons.hardware_control import API, ThreadManager -from opentrons.hardware_control.emulation.app import ( - MAGDECK_PORT, - TEMPDECK_PORT, - THERMOCYCLER_PORT, - SMOOTHIE_PORT, -) from g_code_parsing.g_code_program.g_code_program import ( GCodeProgram, ) @@ -47,9 +48,9 @@ class GCodeEngine: URI_TEMPLATE = "socket://127.0.0.1:%s" - def __init__(self, smoothie_config: Settings) -> None: - self._config = smoothie_config - self._set_env_vars() + def __init__(self, emulator_settings: Settings) -> None: + self._config = emulator_settings + self._set_env_vars(emulator_settings) @staticmethod def _get_loop() -> asyncio.AbstractEventLoop: @@ -62,35 +63,58 @@ def _get_loop() -> asyncio.AbstractEventLoop: return asyncio.get_event_loop() @staticmethod - def _set_env_vars() -> None: + def _set_env_vars(settings: Settings) -> None: """Set URLs of where to find modules and config for smoothie""" - os.environ["OT_MAGNETIC_EMULATOR_URI"] = GCodeEngine.URI_TEMPLATE % MAGDECK_PORT + os.environ["OT_MAGNETIC_EMULATOR_URI"] = GCodeEngine.URI_TEMPLATE % settings.magdeck_proxy.driver_port os.environ["OT_THERMOCYCLER_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % THERMOCYCLER_PORT + GCodeEngine.URI_TEMPLATE % settings.thermocycler_proxy.driver_port ) os.environ["OT_TEMPERATURE_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % TEMPDECK_PORT + GCodeEngine.URI_TEMPLATE % settings.temperature_proxy.driver_port ) @staticmethod - def _start_emulation_app(server_manager: ServerManager) -> None: + def _start_emulation_app(application: Application, emulator_settings: Settings) -> None: """Start emulated OT-2""" + async def _run_emulation_environment() -> None: + await asyncio.gather( + # Start application + Application(settings=emulator_settings).run(), + # Add magdeck emulator + run_emulator_client( + host="localhost", + port=emulator_settings.magdeck_proxy.emulator_port, + emulator=MagDeckEmulator(Parser()), + ), + # Add temperature emulator + run_emulator_client( + host="localhost", + port=emulator_settings.temperature_proxy.emulator_port, + emulator=TempDeckEmulator(Parser()), + ), + # Add thermocycler emulator + run_emulator_client( + host="localhost", + port=emulator_settings.thermocycler_proxy.emulator_port, + emulator=ThermocyclerEmulator(Parser()), + ), + ) def runit(): - asyncio.run(server_manager.run()) + asyncio.run(_run_emulation_environment()) t = threading.Thread(target=runit) t.daemon = True t.start() @staticmethod - def _emulate_hardware() -> ThreadManager: + def _emulate_hardware(settings: Settings) -> ThreadManager: """Created emulated smoothie""" conf = build_config({}) emulator = ThreadManager( API.build_hardware_controller, conf, - GCodeEngine.URI_TEMPLATE % SMOOTHIE_PORT, + GCodeEngine.URI_TEMPLATE % settings.smoothie.port, ) return emulator @@ -111,20 +135,20 @@ def run_protocol(self, path: str) -> Generator: :return: GCodeProgram with all the parsed data """ file_path = os.path.join(get_configuration_dir(), path) - server_manager = ServerManager(self._config) - self._start_emulation_app(server_manager) + emulator_app = Application(self._config) + self._start_emulation_app(application=emulator_app, emulator_settings=self._config) protocol = self._get_protocol(file_path) context = ProtocolContext( implementation=ProtocolContextImplementation( - hardware=self._emulate_hardware() + hardware=self._emulate_hardware(settings=self._config) ), loop=self._get_loop(), ) parsed_protocol = parse(protocol.text, protocol.filename) - with GCodeWatcher() as watcher: + with GCodeWatcher(emulator_settings=self._config) as watcher: execute.run_protocol(parsed_protocol, context=context) yield GCodeProgram.from_g_code_watcher(watcher) - server_manager.stop() + # emulator_app.stop() @contextmanager def run_http(self, executable: Callable): @@ -133,9 +157,9 @@ def run_http(self, executable: Callable): :param executable: Function connected to HTTP Request to execute :return: """ - server_manager = ServerManager(self._config) - self._start_emulation_app(server_manager) - with GCodeWatcher() as watcher: - asyncio.run(executable(hardware=self._emulate_hardware())) + emulator_app = Application(self._config) + self._start_emulation_app(application=emulator_app, emulator_settings=self._config) + with GCodeWatcher(emulator_settings=self._config) as watcher: + asyncio.run(executable(hardware=self._emulate_hardware(settings=self._config))) yield GCodeProgram.from_g_code_watcher(watcher) - server_manager.stop() + # emulator_app.stop() diff --git a/g-code-testing/g_code_parsing/g_code_watcher.py b/g-code-testing/g_code_parsing/g_code_watcher.py index ca137cd28a2..c5d4c88b575 100644 --- a/g-code-testing/g_code_parsing/g_code_watcher.py +++ b/g-code-testing/g_code_parsing/g_code_watcher.py @@ -2,12 +2,8 @@ from typing import List, Optional from opentrons.drivers.asyncio.communication import SerialConnection from dataclasses import dataclass -from opentrons.hardware_control.emulation.app import ( - TEMPDECK_PORT, - THERMOCYCLER_PORT, - SMOOTHIE_PORT, - MAGDECK_PORT, -) + +from opentrons.hardware_control.emulation.settings import Settings @dataclass @@ -23,16 +19,15 @@ class GCodeWatcher: extracts the parameters passed and stores them """ - DEVICE_LOOKUP_BY_PORT = { - SMOOTHIE_PORT: "smoothie", - TEMPDECK_PORT: "tempdeck", - THERMOCYCLER_PORT: "thermocycler", - MAGDECK_PORT: "magdeck", - } - - def __init__(self) -> None: + def __init__(self, emulator_settings: Settings) -> None: self._command_list: List[WatcherData] = [] self._original_send_data = SerialConnection.send_data + self._device_lookup_by_port = { + emulator_settings.smoothie.port: "smoothie", + emulator_settings.temperature_proxy.driver_port: "tempdeck", + emulator_settings.thermocycler_proxy.driver_port: "thermocycler", + emulator_settings.magdeck_proxy.driver_port: "magdeck", + } def __enter__(self) -> GCodeWatcher: """Patch the send command function""" @@ -73,15 +68,14 @@ def __exit__(self, exc_type, exc_val, exc_tb): # mypy error "Cannot assign to a method" is ignored SerialConnection.send_data = self._original_send_data # type: ignore - @classmethod - def _parse_device(cls, serial_connection: SerialConnection): + def _parse_device(self, serial_connection: SerialConnection): """ Based on port specified in connection URL, parse out what the name of the device is """ serial_port = serial_connection.port device_port = serial_port[serial_port.rfind(":") + 1 :] - return cls.DEVICE_LOOKUP_BY_PORT[int(device_port)] + return self._device_lookup_by_port[int(device_port)] def get_command_list(self) -> List[WatcherData]: return self._command_list diff --git a/g-code-testing/tests/g_code_parsing/test_g_code_program.py b/g-code-testing/tests/g_code_parsing/test_g_code_program.py index 8326521c9b5..64f5804d8e0 100644 --- a/g-code-testing/tests/g_code_parsing/test_g_code_program.py +++ b/g-code-testing/tests/g_code_parsing/test_g_code_program.py @@ -1,6 +1,8 @@ -from typing import Generator +from typing import Generator, Iterator import pytest +from opentrons.hardware_control.emulation.settings import Settings + from g_code_parsing import g_code_watcher from g_code_parsing.g_code import GCode from g_code_parsing.g_code_program.g_code_program import ( @@ -10,7 +12,7 @@ @pytest.fixture -def watcher() -> Generator: +def watcher() -> Iterator[g_code_watcher.GCodeWatcher]: def temp_return(self): return [ g_code_watcher.WatcherData("M400", "smoothie", "ok\r\nok\r\n"), @@ -32,7 +34,8 @@ def temp_return(self): old_function = g_code_watcher.GCodeWatcher.get_command_list g_code_watcher.GCodeWatcher.get_command_list = temp_return # type: ignore - yield g_code_watcher.GCodeWatcher() + + yield g_code_watcher.GCodeWatcher(emulator_settings=Settings()) g_code_watcher.GCodeWatcher = old_function # type: ignore From e660db325fb9fa204c40347dc1287052a741cb92 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Wed, 20 Oct 2021 13:58:48 -0400 Subject: [PATCH 07/16] gcode parsing uses process for emulator. --- .../g_code_parsing/g_code_engine.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index a6bd6f778ba..5d068d8c4c8 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -2,6 +2,8 @@ import sys import threading import asyncio +import time +from multiprocessing import Process from typing import Generator, Callable from collections import namedtuple @@ -74,12 +76,12 @@ def _set_env_vars(settings: Settings) -> None: ) @staticmethod - def _start_emulation_app(application: Application, emulator_settings: Settings) -> None: + def _start_emulation_app(application: Application, emulator_settings: Settings) -> Process: """Start emulated OT-2""" async def _run_emulation_environment() -> None: await asyncio.gather( # Start application - Application(settings=emulator_settings).run(), + application.run(), # Add magdeck emulator run_emulator_client( host="localhost", @@ -103,9 +105,10 @@ async def _run_emulation_environment() -> None: def runit(): asyncio.run(_run_emulation_environment()) - t = threading.Thread(target=runit) + t = Process(target=runit) t.daemon = True t.start() + return t @staticmethod def _emulate_hardware(settings: Settings) -> ThreadManager: @@ -136,7 +139,7 @@ def run_protocol(self, path: str) -> Generator: """ file_path = os.path.join(get_configuration_dir(), path) emulator_app = Application(self._config) - self._start_emulation_app(application=emulator_app, emulator_settings=self._config) + app_process = self._start_emulation_app(application=emulator_app, emulator_settings=self._config) protocol = self._get_protocol(file_path) context = ProtocolContext( implementation=ProtocolContextImplementation( @@ -148,7 +151,8 @@ def run_protocol(self, path: str) -> Generator: with GCodeWatcher(emulator_settings=self._config) as watcher: execute.run_protocol(parsed_protocol, context=context) yield GCodeProgram.from_g_code_watcher(watcher) - # emulator_app.stop() + app_process.terminate() + app_process.join() @contextmanager def run_http(self, executable: Callable): @@ -158,8 +162,9 @@ def run_http(self, executable: Callable): :return: """ emulator_app = Application(self._config) - self._start_emulation_app(application=emulator_app, emulator_settings=self._config) + app_process = self._start_emulation_app(application=emulator_app, emulator_settings=self._config) with GCodeWatcher(emulator_settings=self._config) as watcher: asyncio.run(executable(hardware=self._emulate_hardware(settings=self._config))) yield GCodeProgram.from_g_code_watcher(watcher) - # emulator_app.stop() + app_process.terminate() + app_process.join() From 29d269365b12ce0aedf42186d30081aa2a49159a Mon Sep 17 00:00:00 2001 From: amit lissack Date: Thu, 21 Oct 2021 10:56:49 -0400 Subject: [PATCH 08/16] run_emulator_client retries. --- .../hardware_control/emulation/app.py | 17 ++++++-- .../emulation/module_server.py | 2 +- .../emulation/run_emulator.py | 42 +++++++++++++++++-- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/app.py b/api/src/opentrons/hardware_control/emulation/app.py index fe09ff2076c..74de5960aa5 100644 --- a/api/src/opentrons/hardware_control/emulation/app.py +++ b/api/src/opentrons/hardware_control/emulation/app.py @@ -1,5 +1,6 @@ import asyncio import logging +from enum import Enum from opentrons.hardware_control.emulation.module_server import ModuleStatusServer from opentrons.hardware_control.emulation.parser import Parser @@ -11,6 +12,14 @@ logger = logging.getLogger(__name__) +class ModuleType(str, Enum): + """Module type enumeration.""" + Magnetic = "magnetic" + Temperature = "temperature" + Thermocycler = "thermocycler" + Heatershaker = "heatershaker" + + class Application: """The emulator application.""" @@ -26,16 +35,16 @@ def __init__(self, settings: Settings) -> None: parser=Parser(), settings=settings.smoothie ) self._magdeck = Proxy( - "magdeck", self._status_server, self._settings.magdeck_proxy + ModuleType.Magnetic, self._status_server, self._settings.magdeck_proxy ) self._temperature = Proxy( - "temperature", self._status_server, self._settings.temperature_proxy + ModuleType.Temperature, self._status_server, self._settings.temperature_proxy ) self._thermocycler = Proxy( - "thermocycler", self._status_server, self._settings.thermocycler_proxy + ModuleType.Thermocycler, self._status_server, self._settings.thermocycler_proxy ) self._heatershaker = Proxy( - "heatershaker", self._status_server, self._settings.heatershaker_proxy + ModuleType.Heatershaker, self._status_server, self._settings.heatershaker_proxy ) async def run(self) -> None: diff --git a/api/src/opentrons/hardware_control/emulation/module_server.py b/api/src/opentrons/hardware_control/emulation/module_server.py index 0a6443899d9..ba59ca3faa2 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server.py +++ b/api/src/opentrons/hardware_control/emulation/module_server.py @@ -73,7 +73,7 @@ def on_server_disconnected(self, identifier: str) -> None: log.exception("Failed to find identifier") async def run(self) -> None: - """""" + """Run the server.""" server = await asyncio.start_server( self._handle_connection, host=self._settings.host, port=self._settings.port ) diff --git a/api/src/opentrons/hardware_control/emulation/run_emulator.py b/api/src/opentrons/hardware_control/emulation/run_emulator.py index 2ed1a21a696..bd2a74e0603 100644 --- a/api/src/opentrons/hardware_control/emulation/run_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/run_emulator.py @@ -1,5 +1,6 @@ import asyncio import logging +from typing import Optional from opentrons.hardware_control.emulation.abstract_emulator import AbstractEmulator from opentrons.hardware_control.emulation.connection_handler import ConnectionHandler @@ -7,16 +8,49 @@ log = logging.getLogger(__name__) -async def run_emulator_client(host: str, port: int, emulator: AbstractEmulator) -> None: - """Run an emulator as a client.""" +async def run_emulator_client(host: str, port: int, emulator: AbstractEmulator, retries: int = 3, interval_seconds:float=0.1) -> None: + """Run an emulator as a client. + + Args: + host: Host to connect to. + port: Port to connect on. + emulator: The emulator instance. + retries: Number of retries on a failed connection attempt. + interval_seconds: How long to wait between retries. + + Returns: + None + """ log.info(f"Connecting to {emulator.__class__.__name__} at {host}:{port}") - r, w = await asyncio.open_connection(host, port) + + r: Optional[asyncio.StreamReader] = None + w: Optional[asyncio.StreamWriter] = None + for i in range(retries): + try: + r, w = await asyncio.open_connection(host, port) + break + except IOError: + log.error(f"{emulator.__class__.__name__} failed to connect on try {i + 1}. Retrying in {interval_seconds} seconds.") + await asyncio.sleep(interval_seconds) + + if r is None or w is None: + raise IOError(f"Failed to connect to {emulator.__class__.__name__} at {host}:{port} after {retries} retries.") + connection = ConnectionHandler(emulator) await connection(r, w) async def run_emulator_server(host: str, port: int, emulator: AbstractEmulator) -> None: - """Run an emulator as a server.""" + """Run an emulator as a server. + + Args: + host: Host to listen on. + port: Port to listen on. + emulator: Emaulator instance. + + Returns: + None + """ log.info(f"Starting {emulator.__class__.__name__} at {host}:{port}") server = await asyncio.start_server(ConnectionHandler(emulator), host, port) await server.serve_forever() From cf47d68f297c045c690435ce9a1169ca16a8ebf1 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Thu, 21 Oct 2021 12:53:43 -0400 Subject: [PATCH 09/16] wait for emulator connection method. --- .../hardware_control/emulation/app.py | 13 ++- .../emulation/module_server.py | 107 +++++++++++++++++- .../emulation/run_emulator.py | 16 ++- .../hardware_control/integration/conftest.py | 2 - 4 files changed, 126 insertions(+), 12 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/app.py b/api/src/opentrons/hardware_control/emulation/app.py index 74de5960aa5..f71ad647c08 100644 --- a/api/src/opentrons/hardware_control/emulation/app.py +++ b/api/src/opentrons/hardware_control/emulation/app.py @@ -14,6 +14,7 @@ class ModuleType(str, Enum): """Module type enumeration.""" + Magnetic = "magnetic" Temperature = "temperature" Thermocycler = "thermocycler" @@ -38,13 +39,19 @@ def __init__(self, settings: Settings) -> None: ModuleType.Magnetic, self._status_server, self._settings.magdeck_proxy ) self._temperature = Proxy( - ModuleType.Temperature, self._status_server, self._settings.temperature_proxy + ModuleType.Temperature, + self._status_server, + self._settings.temperature_proxy, ) self._thermocycler = Proxy( - ModuleType.Thermocycler, self._status_server, self._settings.thermocycler_proxy + ModuleType.Thermocycler, + self._status_server, + self._settings.thermocycler_proxy, ) self._heatershaker = Proxy( - ModuleType.Heatershaker, self._status_server, self._settings.heatershaker_proxy + ModuleType.Heatershaker, + self._status_server, + self._settings.heatershaker_proxy, ) async def run(self) -> None: diff --git a/api/src/opentrons/hardware_control/emulation/module_server.py b/api/src/opentrons/hardware_control/emulation/module_server.py index ba59ca3faa2..232d5aa287f 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server.py +++ b/api/src/opentrons/hardware_control/emulation/module_server.py @@ -2,8 +2,10 @@ import asyncio import logging -from typing_extensions import Literal -from typing import Dict, List, Set + +from opentrons.hardware_control.emulation.app import ModuleType +from typing_extensions import Literal, Final +from typing import Dict, List, Set, Sequence, Optional from pydantic import BaseModel from opentrons.hardware_control.emulation.proxy import ProxyListener @@ -12,6 +14,8 @@ log = logging.getLogger(__name__) +MessageDelimiter: Final = b"\n" + class ModuleStatusServer(ProxyListener): """Server notifying of module connections.""" @@ -68,7 +72,7 @@ def on_server_disconnected(self, identifier: str) -> None: .json() .encode() ) - c.write(b"\n") + c.write(MessageDelimiter) except KeyError: log.exception("Failed to find identifier") @@ -88,7 +92,7 @@ async def _handle_connection( m = Message(status="dump", connections=list(self._connections.values())) writer.write(m.json().encode()) - writer.write(b"\n") + writer.write(MessageDelimiter) self._clients.add(writer) @@ -97,6 +101,8 @@ async def _handle_connection( self._clients.remove(writer) break + log.info("Client disconnected from module server.") + class Connection(BaseModel): """Model a single module connection.""" @@ -111,3 +117,96 @@ class Message(BaseModel): status: Literal["connected", "disconnected", "dump"] connections: List[Connection] + + +class ModuleServerClient: + """A module server client.""" + + def __init__( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """Constructor.""" + self._reader = reader + self._writer = writer + + @classmethod + async def connect( + cls, + host: str, + port: int, + retries: int = 3, + interval_seconds: float = 0.1, + ) -> ModuleServerClient: + """Connect to the module server. + + Args: + host: module server host. + port: module server port. + retries: number of retries + interval_seconds: time between retries. + + Returns: + None + Raises: + IOError on retry expiry. + """ + r: Optional[asyncio.StreamReader] = None + w: Optional[asyncio.StreamWriter] = None + for i in range(retries): + try: + r, w = await asyncio.open_connection(host=host, port=port) + except OSError: + await asyncio.sleep(interval_seconds) + + if r is not None and w is not None: + return ModuleServerClient(reader=r, writer=w) + else: + raise IOError( + f"Failed to connect to module_server at after {retries} retries." + ) + + async def read(self) -> Message: + """Read a message from the module server.""" + b = await self._reader.readuntil(MessageDelimiter) + m: Message = Message.parse_raw(b) + return m + + +async def wait_emulators( + client: ModuleServerClient, + modules: Sequence[ModuleType], + timeout: float, +) -> None: + """Wait for module emulators to connect. + + Args: + client: module server client. + modules: collection of of module types to wait for. + timeout: how long to wait for emulators to connect (in seconds) + + Returns: + None + Raises: + asyncio.TimeoutError on timeout. + """ + + async def _wait_modules(cl: ModuleServerClient, module_set: Set[str]) -> None: + """Read messages from module server waiting for modules in module_set to + be connected.""" + while module_set: + m: Message = await cl.read() + if m.status == "dump" or m.status == "connected": + for c in m.connections: + if c.module_type in module_set: + module_set.remove(c.module_type) + elif m.status == "disconnected": + for c in m.connections: + if c.module_type in module_set: + module_set.add(c.module_type) + + log.debug(f"after message: {m}, awaiting module set is: {module_set}") + + await asyncio.wait_for( + _wait_modules(cl=client, module_set=set(n.value for n in modules)), + timeout=timeout, + ) diff --git a/api/src/opentrons/hardware_control/emulation/run_emulator.py b/api/src/opentrons/hardware_control/emulation/run_emulator.py index bd2a74e0603..25bee731082 100644 --- a/api/src/opentrons/hardware_control/emulation/run_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/run_emulator.py @@ -8,7 +8,13 @@ log = logging.getLogger(__name__) -async def run_emulator_client(host: str, port: int, emulator: AbstractEmulator, retries: int = 3, interval_seconds:float=0.1) -> None: +async def run_emulator_client( + host: str, + port: int, + emulator: AbstractEmulator, + retries: int = 3, + interval_seconds: float = 0.1, +) -> None: """Run an emulator as a client. Args: @@ -30,11 +36,15 @@ async def run_emulator_client(host: str, port: int, emulator: AbstractEmulator, r, w = await asyncio.open_connection(host, port) break except IOError: - log.error(f"{emulator.__class__.__name__} failed to connect on try {i + 1}. Retrying in {interval_seconds} seconds.") + log.error( + f"{emulator.__class__.__name__} failed to connect on try {i + 1}. Retrying in {interval_seconds} seconds." + ) await asyncio.sleep(interval_seconds) if r is None or w is None: - raise IOError(f"Failed to connect to {emulator.__class__.__name__} at {host}:{port} after {retries} retries.") + raise IOError( + f"Failed to connect to {emulator.__class__.__name__} at {host}:{port} after {retries} retries." + ) connection = ConnectionHandler(emulator) await connection(r, w) diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 91cac71aca0..86e401e0f68 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -67,6 +67,4 @@ def runit() -> None: t = threading.Thread(target=runit) t.daemon = True t.start() - # Give it a bit to get going. - sleep(0.5) yield From 173827787e8da8d20b20e28950d2347914d00ac9 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Fri, 22 Oct 2021 09:55:58 -0400 Subject: [PATCH 10/16] scripts. --- .../hardware_control/emulation/app.py | 11 +--- .../emulation/module_server.py | 2 +- .../emulation/run_emulator.py | 6 +- .../emulation/run_module_emulator.py | 52 --------------- .../emulation/scripts/__init__.py | 0 .../emulation/scripts/run_app.py | 54 +++++++++++++++ .../emulation/scripts/run_module_emulator.py | 65 +++++++++++++++++++ .../hardware_control/emulation/types.py | 10 +++ .../hardware_control/integration/conftest.py | 44 ++++--------- 9 files changed, 147 insertions(+), 97 deletions(-) delete mode 100644 api/src/opentrons/hardware_control/emulation/run_module_emulator.py create mode 100644 api/src/opentrons/hardware_control/emulation/scripts/__init__.py create mode 100644 api/src/opentrons/hardware_control/emulation/scripts/run_app.py create mode 100644 api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py create mode 100644 api/src/opentrons/hardware_control/emulation/types.py diff --git a/api/src/opentrons/hardware_control/emulation/app.py b/api/src/opentrons/hardware_control/emulation/app.py index f71ad647c08..00a11db5f58 100644 --- a/api/src/opentrons/hardware_control/emulation/app.py +++ b/api/src/opentrons/hardware_control/emulation/app.py @@ -1,6 +1,5 @@ import asyncio import logging -from enum import Enum from opentrons.hardware_control.emulation.module_server import ModuleStatusServer from opentrons.hardware_control.emulation.parser import Parser @@ -8,19 +7,11 @@ from opentrons.hardware_control.emulation.run_emulator import run_emulator_server from opentrons.hardware_control.emulation.settings import Settings from opentrons.hardware_control.emulation.smoothie import SmoothieEmulator +from opentrons.hardware_control.emulation.types import ModuleType logger = logging.getLogger(__name__) -class ModuleType(str, Enum): - """Module type enumeration.""" - - Magnetic = "magnetic" - Temperature = "temperature" - Thermocycler = "thermocycler" - Heatershaker = "heatershaker" - - class Application: """The emulator application.""" diff --git a/api/src/opentrons/hardware_control/emulation/module_server.py b/api/src/opentrons/hardware_control/emulation/module_server.py index 232d5aa287f..a7aa7b5e87c 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server.py +++ b/api/src/opentrons/hardware_control/emulation/module_server.py @@ -3,7 +3,7 @@ import asyncio import logging -from opentrons.hardware_control.emulation.app import ModuleType +from opentrons.hardware_control.emulation.types import ModuleType from typing_extensions import Literal, Final from typing import Dict, List, Set, Sequence, Optional from pydantic import BaseModel diff --git a/api/src/opentrons/hardware_control/emulation/run_emulator.py b/api/src/opentrons/hardware_control/emulation/run_emulator.py index 25bee731082..aedeec27e63 100644 --- a/api/src/opentrons/hardware_control/emulation/run_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/run_emulator.py @@ -37,13 +37,15 @@ async def run_emulator_client( break except IOError: log.error( - f"{emulator.__class__.__name__} failed to connect on try {i + 1}. Retrying in {interval_seconds} seconds." + f"{emulator.__class__.__name__} failed to connect on " + f"try {i + 1}. Retrying in {interval_seconds} seconds." ) await asyncio.sleep(interval_seconds) if r is None or w is None: raise IOError( - f"Failed to connect to {emulator.__class__.__name__} at {host}:{port} after {retries} retries." + f"Failed to connect to {emulator.__class__.__name__} at " + f"{host}:{port} after {retries} retries." ) connection = ConnectionHandler(emulator) diff --git a/api/src/opentrons/hardware_control/emulation/run_module_emulator.py b/api/src/opentrons/hardware_control/emulation/run_module_emulator.py deleted file mode 100644 index 51fa5c8bfc8..00000000000 --- a/api/src/opentrons/hardware_control/emulation/run_module_emulator.py +++ /dev/null @@ -1,52 +0,0 @@ -import logging -import asyncio -from argparse import ArgumentParser -from typing import Dict, Callable - -from opentrons.hardware_control.emulation.abstract_emulator import AbstractEmulator -from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator -from opentrons.hardware_control.emulation.parser import Parser -from opentrons.hardware_control.emulation.run_emulator import run_emulator_client -from opentrons.hardware_control.emulation.settings import Settings, ProxySettings -from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator -from opentrons.hardware_control.emulation.thermocycler import ThermocyclerEmulator - -emulator_builder: Dict[str, Callable[[Settings], AbstractEmulator]] = { - "magdeck": lambda s: MagDeckEmulator(Parser()), - "temperature": lambda s: TempDeckEmulator(Parser()), - "thermocycler": lambda s: ThermocyclerEmulator(Parser()), -} - -emulator_port: Dict[str, Callable[[Settings], ProxySettings]] = { - "magdeck": lambda s: s.magdeck_proxy, - "temperature": lambda s: s.temperature_proxy, - "thermocycler": lambda s: s.thermocycler_proxy, -} - - -def run(emulator_name: str, host: str) -> None: - """Run an emulator. - - Args: - emulator_name: Name of emulator. This must be a key in emulator_builder - host: host to connect to. - - Returns: - None - """ - settings = Settings() - - e = emulator_builder[emulator_name](settings) - proxy_settings = emulator_port[emulator_name](settings) - - asyncio.run(run_emulator_client(host, proxy_settings.emulator_port, e)) - - -if __name__ == "__main__": - a = ArgumentParser() - a.add_argument("emulator", type=str, choices=emulator_builder.keys()) - a.add_argument("host", type=str) - args = a.parse_args() - - logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) - run(args.emulator, args.host) diff --git a/api/src/opentrons/hardware_control/emulation/scripts/__init__.py b/api/src/opentrons/hardware_control/emulation/scripts/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/src/opentrons/hardware_control/emulation/scripts/run_app.py b/api/src/opentrons/hardware_control/emulation/scripts/run_app.py new file mode 100644 index 00000000000..47559487691 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/scripts/run_app.py @@ -0,0 +1,54 @@ +"""Script for starting up emulation up with module emulators.""" +import logging +import asyncio +from argparse import ArgumentParser +from typing import List + +from opentrons.hardware_control.emulation.app import Application +from opentrons.hardware_control.emulation.scripts.run_module_emulator import ( + emulator_builder, +) +from opentrons.hardware_control.emulation.settings import Settings +from .run_module_emulator import run as run_module_by_name + + +async def run(settings: Settings, modules: List[str]) -> None: + """Run the emulator app with connected module emulators. + + Args: + settings: App settings. + modules: The module emulators to start. + + Returns: + None + + """ + loop = asyncio.get_event_loop() + + app_task = loop.create_task(Application(settings=settings).run()) + module_tasks = [ + loop.create_task( + run_module_by_name(settings=settings, emulator_name=n, host="localhost") + ) + for n in modules + ] + await asyncio.gather(app_task, *module_tasks) + + +def main() -> None: + """Entry point.""" + a = ArgumentParser() + a.add_argument( + "--m", + action="append", + choices=emulator_builder.keys(), + help="which module(s) to emulate.", + ) + args = a.parse_args() + + logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) + asyncio.run(run(Settings(), args.m)) + + +if __name__ == "__main__": + main() diff --git a/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py b/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py new file mode 100644 index 00000000000..4c19fe539d2 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py @@ -0,0 +1,65 @@ +"""Script for starting up a python module emulator.""" +import logging +import asyncio +from argparse import ArgumentParser +from typing import Dict, Callable +from typing_extensions import Final + +from opentrons.hardware_control.emulation.abstract_emulator import AbstractEmulator +from opentrons.hardware_control.emulation.types import ModuleType +from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator +from opentrons.hardware_control.emulation.parser import Parser +from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator +from opentrons.hardware_control.emulation.thermocycler import ThermocyclerEmulator + + +from opentrons.hardware_control.emulation.run_emulator import run_emulator_client +from opentrons.hardware_control.emulation.settings import Settings, ProxySettings + +emulator_builder: Final[Dict[str, Callable[[Settings], AbstractEmulator]]] = { + ModuleType.Magnetic.value: lambda s: MagDeckEmulator(Parser()), + ModuleType.Temperature.value: lambda s: TempDeckEmulator(Parser()), + ModuleType.Thermocycler.value: lambda s: ThermocyclerEmulator(Parser()), +} + +emulator_port: Final[Dict[str, Callable[[Settings], ProxySettings]]] = { + ModuleType.Magnetic.value: lambda s: s.magdeck_proxy, + ModuleType.Temperature.value: lambda s: s.temperature_proxy, + ModuleType.Thermocycler.value: lambda s: s.thermocycler_proxy, +} + + +async def run(settings: Settings, emulator_name: str, host: str) -> None: + """Run an emulator. + + Args: + settings: emulator settings + emulator_name: Name of emulator. This must be a key in emulator_builder + host: host to connect to. + + Returns: + None + """ + e = emulator_builder[emulator_name](settings) + proxy_settings = emulator_port[emulator_name](settings) + await run_emulator_client(host, proxy_settings.emulator_port, e) + + +def main() -> None: + """Entry point.""" + a = ArgumentParser() + a.add_argument( + "emulator", + type=str, + choices=emulator_builder.keys(), + help="which module to emulate.", + ) + a.add_argument("host", type=str, help="the emulator host") + args = a.parse_args() + + logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG) + asyncio.run(run(Settings(), args.emulator, args.host)) + + +if __name__ == "__main__": + main() diff --git a/api/src/opentrons/hardware_control/emulation/types.py b/api/src/opentrons/hardware_control/emulation/types.py new file mode 100644 index 00000000000..cbdd3eecb11 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/types.py @@ -0,0 +1,10 @@ +from enum import Enum + + +class ModuleType(str, Enum): + """Module type enumeration.""" + + Magnetic = "magnetic" + Temperature = "temperature" + Thermocycler = "thermocycler" + Heatershaker = "heatershaker" diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 86e401e0f68..3ac09b54640 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -1,20 +1,15 @@ -from time import sleep from typing import Iterator import pytest import threading import asyncio -from opentrons.hardware_control.emulation.app import Application -from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator -from opentrons.hardware_control.emulation.parser import Parser -from opentrons.hardware_control.emulation.run_emulator import run_emulator_client +from opentrons.hardware_control.emulation.scripts import run_app +from opentrons.hardware_control.emulation.types import ModuleType from opentrons.hardware_control.emulation.settings import ( Settings, SmoothieSettings, PipetteSettings, ) -from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator -from opentrons.hardware_control.emulation.thermocycler import ThermocyclerEmulator @pytest.fixture(scope="session") @@ -33,32 +28,17 @@ def emulator_settings() -> Settings: def emulation_app(emulator_settings: Settings) -> Iterator[None]: """Run the emulators""" - async def _run_emulation_environment() -> None: - await asyncio.gather( - # Start application - Application(settings=emulator_settings).run(), - # Add magdeck emulator - run_emulator_client( - host="localhost", - port=emulator_settings.magdeck_proxy.emulator_port, - emulator=MagDeckEmulator(Parser()), - ), - # Add temperature emulator - run_emulator_client( - host="localhost", - port=emulator_settings.temperature_proxy.emulator_port, - emulator=TempDeckEmulator(Parser()), - ), - # Add thermocycler emulator - run_emulator_client( - host="localhost", - port=emulator_settings.thermocycler_proxy.emulator_port, - emulator=ThermocyclerEmulator(Parser()), - ), - ) - def runit() -> None: - asyncio.run(_run_emulation_environment()) + asyncio.run( + run_app.run( + settings=emulator_settings, + modules=[ + ModuleType.Magnetic, + ModuleType.Temperature, + ModuleType.Thermocycler, + ], + ) + ) # TODO 20210219 # The emulators must be run in a separate thread because our serial From acc6dcdf1a607c51826b438aaf0c931748ad7f82 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Fri, 22 Oct 2021 11:27:08 -0400 Subject: [PATCH 11/16] utils --- .../emulation/module_server.py | 4 ++ .../emulation/run_emulator.py | 2 +- .../g_code_parsing/g_code_engine.py | 56 ++++++++----------- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/api/src/opentrons/hardware_control/emulation/module_server.py b/api/src/opentrons/hardware_control/emulation/module_server.py index a7aa7b5e87c..2a09a21afd2 100644 --- a/api/src/opentrons/hardware_control/emulation/module_server.py +++ b/api/src/opentrons/hardware_control/emulation/module_server.py @@ -171,6 +171,10 @@ async def read(self) -> Message: m: Message = Message.parse_raw(b) return m + def close(self) -> None: + """Close the client.""" + self._writer.close() + async def wait_emulators( client: ModuleServerClient, diff --git a/api/src/opentrons/hardware_control/emulation/run_emulator.py b/api/src/opentrons/hardware_control/emulation/run_emulator.py index aedeec27e63..c4709cec7a8 100644 --- a/api/src/opentrons/hardware_control/emulation/run_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/run_emulator.py @@ -58,7 +58,7 @@ async def run_emulator_server(host: str, port: int, emulator: AbstractEmulator) Args: host: Host to listen on. port: Port to listen on. - emulator: Emaulator instance. + emulator: Emulator instance. Returns: None diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index 5d068d8c4c8..d33b3134e95 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -15,12 +15,14 @@ from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator from opentrons.hardware_control.emulation.thermocycler import \ ThermocyclerEmulator +from opentrons.hardware_control.emulation.types import ModuleType from opentrons.protocols.parse import parse from opentrons.protocols.execution import execute from contextlib import contextmanager from opentrons.protocol_api import ProtocolContext from opentrons.config.robot_configs import build_config -from opentrons.hardware_control.emulation.app import Application +from opentrons.hardware_control.emulation import module_server +from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control import API, ThreadManager from g_code_parsing.g_code_program.g_code_program import ( GCodeProgram, @@ -76,39 +78,27 @@ def _set_env_vars(settings: Settings) -> None: ) @staticmethod - def _start_emulation_app(application: Application, emulator_settings: Settings) -> Process: + def _start_emulation_app(emulator_settings: Settings) -> Process: """Start emulated OT-2""" - async def _run_emulation_environment() -> None: - await asyncio.gather( - # Start application - application.run(), - # Add magdeck emulator - run_emulator_client( - host="localhost", - port=emulator_settings.magdeck_proxy.emulator_port, - emulator=MagDeckEmulator(Parser()), - ), - # Add temperature emulator - run_emulator_client( - host="localhost", - port=emulator_settings.temperature_proxy.emulator_port, - emulator=TempDeckEmulator(Parser()), - ), - # Add thermocycler emulator - run_emulator_client( - host="localhost", - port=emulator_settings.thermocycler_proxy.emulator_port, - emulator=ThermocyclerEmulator(Parser()), - ), - ) + modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] def runit(): - asyncio.run(_run_emulation_environment()) + asyncio.run(run_app.run(emulator_settings, modules=[m.value for m in modules])) - t = Process(target=runit) - t.daemon = True - t.start() - return t + proc = Process(target=runit) + proc.daemon = True + proc.start() + + async def _wait_ready() -> None: + c = await module_server.ModuleServerClient.connect(host="localhost", port=emulator_settings.module_server.port) + await module_server.wait_emulators(client=c, modules=modules, timeout=5) + c.close() + + proc2 = Process(target=lambda: asyncio.run(_wait_ready())) + proc2.start() + proc2.join() + + return proc @staticmethod def _emulate_hardware(settings: Settings) -> ThreadManager: @@ -138,8 +128,7 @@ def run_protocol(self, path: str) -> Generator: :return: GCodeProgram with all the parsed data """ file_path = os.path.join(get_configuration_dir(), path) - emulator_app = Application(self._config) - app_process = self._start_emulation_app(application=emulator_app, emulator_settings=self._config) + app_process = self._start_emulation_app(emulator_settings=self._config) protocol = self._get_protocol(file_path) context = ProtocolContext( implementation=ProtocolContextImplementation( @@ -161,8 +150,7 @@ def run_http(self, executable: Callable): :param executable: Function connected to HTTP Request to execute :return: """ - emulator_app = Application(self._config) - app_process = self._start_emulation_app(application=emulator_app, emulator_settings=self._config) + app_process = self._start_emulation_app(emulator_settings=self._config) with GCodeWatcher(emulator_settings=self._config) as watcher: asyncio.run(executable(hardware=self._emulate_hardware(settings=self._config))) yield GCodeProgram.from_g_code_watcher(watcher) From bb7c9661cff54b5a31c13427796c9c5dd4fe8065 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Sat, 23 Oct 2021 10:34:29 -0400 Subject: [PATCH 12/16] g-code-testing lint --- .../g_code_parsing/g_code_engine.py | 25 +++++++++---------- .../g_code_parsing/test_g_code_program.py | 2 +- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index d33b3134e95..36c323cb617 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -1,20 +1,11 @@ import os import sys -import threading import asyncio -import time from multiprocessing import Process from typing import Generator, Callable from collections import namedtuple -from opentrons.hardware_control.emulation.magdeck import MagDeckEmulator -from opentrons.hardware_control.emulation.parser import Parser -from opentrons.hardware_control.emulation.run_emulator import \ - run_emulator_client from opentrons.hardware_control.emulation.settings import Settings -from opentrons.hardware_control.emulation.tempdeck import TempDeckEmulator -from opentrons.hardware_control.emulation.thermocycler import \ - ThermocyclerEmulator from opentrons.hardware_control.emulation.types import ModuleType from opentrons.protocols.parse import parse from opentrons.protocols.execution import execute @@ -69,7 +60,9 @@ def _get_loop() -> asyncio.AbstractEventLoop: @staticmethod def _set_env_vars(settings: Settings) -> None: """Set URLs of where to find modules and config for smoothie""" - os.environ["OT_MAGNETIC_EMULATOR_URI"] = GCodeEngine.URI_TEMPLATE % settings.magdeck_proxy.driver_port + os.environ["OT_MAGNETIC_EMULATOR_URI"] = ( + GCodeEngine.URI_TEMPLATE % settings.magdeck_proxy.driver_port + ) os.environ["OT_THERMOCYCLER_EMULATOR_URI"] = ( GCodeEngine.URI_TEMPLATE % settings.thermocycler_proxy.driver_port ) @@ -83,14 +76,18 @@ def _start_emulation_app(emulator_settings: Settings) -> Process: modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] def runit(): - asyncio.run(run_app.run(emulator_settings, modules=[m.value for m in modules])) + asyncio.run( + run_app.run(emulator_settings, modules=[m.value for m in modules]) + ) proc = Process(target=runit) proc.daemon = True proc.start() async def _wait_ready() -> None: - c = await module_server.ModuleServerClient.connect(host="localhost", port=emulator_settings.module_server.port) + c = await module_server.ModuleServerClient.connect( + host="localhost", port=emulator_settings.module_server.port + ) await module_server.wait_emulators(client=c, modules=modules, timeout=5) c.close() @@ -152,7 +149,9 @@ def run_http(self, executable: Callable): """ app_process = self._start_emulation_app(emulator_settings=self._config) with GCodeWatcher(emulator_settings=self._config) as watcher: - asyncio.run(executable(hardware=self._emulate_hardware(settings=self._config))) + asyncio.run( + executable(hardware=self._emulate_hardware(settings=self._config)) + ) yield GCodeProgram.from_g_code_watcher(watcher) app_process.terminate() app_process.join() diff --git a/g-code-testing/tests/g_code_parsing/test_g_code_program.py b/g-code-testing/tests/g_code_parsing/test_g_code_program.py index 64f5804d8e0..b4f54a1aa9a 100644 --- a/g-code-testing/tests/g_code_parsing/test_g_code_program.py +++ b/g-code-testing/tests/g_code_parsing/test_g_code_program.py @@ -1,4 +1,4 @@ -from typing import Generator, Iterator +from typing import Iterator import pytest from opentrons.hardware_control.emulation.settings import Settings From 0be1cd0628951a37dcb98d3196dc61a88432d8fb Mon Sep 17 00:00:00 2001 From: amitlissack Date: Thu, 4 Nov 2021 13:38:53 -0400 Subject: [PATCH 13/16] refactor(emulation): integrated with module control (#8586) * Module control refactors * new package. * start module control integration. * tests for module control integration. * implement connection listener. * module server integrated into module control. * lint * g-code-testing * redo docker compose to deal with separate emulator apps. * usb port. * expand module emulation settings. * update docker readme. * format-js * lint * go back to threadings. I don't want to debug windows nonsense. * fix bug. * redo gcode testing's emulator setup. * documentation. * clean up. --- DOCKER.md | 56 +++++ api/src/opentrons/hardware_control/api.py | 8 +- .../opentrons/hardware_control/controller.py | 3 +- .../hardware_control/emulation/magdeck.py | 17 +- .../emulation/module_server.py | 216 ------------------ .../emulation/module_server/__init__.py | 8 + .../emulation/module_server/client.py | 72 ++++++ .../emulation/module_server/helpers.py | 129 +++++++++++ .../emulation/module_server/models.py | 31 +++ .../emulation/module_server/server.py | 108 +++++++++ .../hardware_control/emulation/proxy.py | 8 +- .../emulation/scripts/run_module_emulator.py | 8 +- .../hardware_control/emulation/settings.py | 47 +++- .../hardware_control/emulation/tempdeck.py | 24 +- .../emulation/thermocycler.py | 25 +- .../hardware_control/emulation/types.py | 4 +- .../hardware_control/module_control.py | 79 +++---- .../opentrons/hardware_control/simulator.py | 2 - .../emulation/module_server/__init__.py | 0 .../emulation/module_server/test_helpers.py | 152 ++++++++++++ .../hardware_control/integration/conftest.py | 44 ++-- docker-compose.yml | 35 ++- g-code-testing/cli.py | 2 +- .../g_code_parsing/g_code_engine.py | 107 ++++----- .../g_code_test_data/http/http_settings.py | 8 +- .../g_code_test_data/http/modules/magdeck.py | 8 +- .../g_code_test_data/http/modules/tempdeck.py | 8 +- .../http/modules/thermocycler.py | 17 +- .../protocol/protocol_configurations.py | 21 +- .../g_code_parsing/test_g_code_engine.py | 1 - 30 files changed, 834 insertions(+), 414 deletions(-) delete mode 100644 api/src/opentrons/hardware_control/emulation/module_server.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/__init__.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/client.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/helpers.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/models.py create mode 100644 api/src/opentrons/hardware_control/emulation/module_server/server.py create mode 100644 api/tests/opentrons/hardware_control/emulation/module_server/__init__.py create mode 100644 api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py diff --git a/DOCKER.md b/DOCKER.md index 6a02c7f459e..3bb4026b34f 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -37,6 +37,62 @@ For example to use a `p300_multi` on the right add: OT_EMULATOR_smoothie: '{"right": {"model": "p300_multi"}}' ``` +### Adding more emulators + +#### Magdeck + +To add a second mag deck emulator make a copy of the existing `magdeck` section and change the key and `serial_number`. + +For example this adds a `magdeck` with the serial number `magdeck2`: + +``` + magdeck2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator magdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_magdeck: '{"serial_number": "magdeck2", "model":"mag_deck_v20", "version":"2.0.0"}' +``` + +#### Tempdeck + +To add a second temp deck emulator make a copy of the existing `tempdeck` section and change the key and `serial_number`. + +For example this adds a `tempdeck` with the serial number `tempdeck2`: + +``` + tempdeck2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator tempdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_tempdeck: '{"serial_number": "tempdeck2", "model":"temp_deck_v20", "version":"v2.0.1", "temperature": {"starting":0.0, "degrees_per_tick": 2.0}}' +``` + +#### Thermocycler + +To add a second thermocycler emulator make a copy of the existing `thermocycler` section and change the key and `serial_number`. + +For example this adds a `thermocycler` with the serial number `thermocycler2`: + +``` + thermocycler2: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator thermocycler emulator + links: + - 'emulator' + depends_on: + - 'emulator' + environment: + OT_EMULATOR_thermocycler: '{"serial_number": "thermocycler2", "model":"v02", "version":"v1.1.0", "lid_temperature": {"starting":23.0, "degrees_per_tick": 2.0}, "plate_temperature": {"starting":23.0, "degrees_per_tick": 2.0}}' +``` + ## Known Issues - Pipettes cannot be changed at run time. diff --git a/api/src/opentrons/hardware_control/api.py b/api/src/opentrons/hardware_control/api.py index 2bc50c65cb6..350f2cc39e1 100644 --- a/api/src/opentrons/hardware_control/api.py +++ b/api/src/opentrons/hardware_control/api.py @@ -214,7 +214,9 @@ async def blink(): api_instance = cls(backend, loop=checked_loop, config=checked_config) await api_instance.cache_instruments() - module_controls = await AttachedModulesControl.build(api_instance) + module_controls = await AttachedModulesControl.build( + api_instance, board_revision=backend.board_revision + ) backend.module_controls = module_controls checked_loop.create_task(backend.watch(loop=checked_loop)) backend.start_gpio_door_watcher( @@ -260,7 +262,9 @@ async def build_hardware_simulator( ) api_instance = cls(backend, loop=checked_loop, config=checked_config) await api_instance.cache_instruments() - module_controls = await AttachedModulesControl.build(api_instance) + module_controls = await AttachedModulesControl.build( + api_instance, board_revision=backend.board_revision + ) backend.module_controls = module_controls await backend.watch() return api_instance diff --git a/api/src/opentrons/hardware_control/controller.py b/api/src/opentrons/hardware_control/controller.py index 909ffaf5e64..e63be888731 100644 --- a/api/src/opentrons/hardware_control/controller.py +++ b/api/src/opentrons/hardware_control/controller.py @@ -11,7 +11,7 @@ aionotify = None from opentrons.drivers.smoothie_drivers import SmoothieDriver -from opentrons.drivers.rpi_drivers import build_gpio_chardev, usb +from opentrons.drivers.rpi_drivers import build_gpio_chardev import opentrons.config from opentrons.config import pipette_config from opentrons.config.types import RobotConfig @@ -77,7 +77,6 @@ def __init__(self, config: RobotConfig, gpio: GPIODriverLike): config=self.config, gpio_chardev=self._gpio_chardev ) self._cached_fw_version: Optional[str] = None - self._usb = usb.USBBus(self._board_revision) self._module_controls: Optional[AttachedModulesControl] = None try: self._event_watcher = self._build_event_watcher() diff --git a/api/src/opentrons/hardware_control/emulation/magdeck.py b/api/src/opentrons/hardware_control/emulation/magdeck.py index bfcc8d39587..eaa7eb718d1 100644 --- a/api/src/opentrons/hardware_control/emulation/magdeck.py +++ b/api/src/opentrons/hardware_control/emulation/magdeck.py @@ -8,24 +8,21 @@ from opentrons.drivers.mag_deck.driver import GCODE from opentrons.hardware_control.emulation.parser import Parser, Command from .abstract_emulator import AbstractEmulator +from .settings import MagDeckSettings logger = logging.getLogger(__name__) -SERIAL = "magnetic_emulator" -MODEL = "mag_deck_v20" -VERSION = "2.0.0" - - class MagDeckEmulator(AbstractEmulator): """Magdeck emulator""" height: float = 0 position: float = 0 - def __init__(self, parser: Parser) -> None: - self.reset() + def __init__(self, parser: Parser, settings: MagDeckSettings) -> None: + self._settings = settings self._parser = parser + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -53,7 +50,11 @@ def _handle(self, command: Command) -> Optional[str]: elif command.gcode == GCODE.GET_CURRENT_POSITION: return f"Z:{self.position}" elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) elif command.gcode == GCODE.PROGRAMMING_MODE: pass return None diff --git a/api/src/opentrons/hardware_control/emulation/module_server.py b/api/src/opentrons/hardware_control/emulation/module_server.py deleted file mode 100644 index 2a09a21afd2..00000000000 --- a/api/src/opentrons/hardware_control/emulation/module_server.py +++ /dev/null @@ -1,216 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging - -from opentrons.hardware_control.emulation.types import ModuleType -from typing_extensions import Literal, Final -from typing import Dict, List, Set, Sequence, Optional -from pydantic import BaseModel - -from opentrons.hardware_control.emulation.proxy import ProxyListener -from opentrons.hardware_control.emulation.settings import ModuleServerSettings - - -log = logging.getLogger(__name__) - -MessageDelimiter: Final = b"\n" - - -class ModuleStatusServer(ProxyListener): - """Server notifying of module connections.""" - - def __init__(self, settings: ModuleServerSettings) -> None: - """Constructor - - Args: - settings: app settings - """ - self._settings = settings - self._connections: Dict[str, Connection] = {} - self._clients: Set[asyncio.StreamWriter] = set() - - def on_server_connected( - self, server_type: str, client_uri: str, identifier: str - ) -> None: - """Called when a new module has connected. - - Args: - server_type: the type of module - client_uri: the url string for a driver to connect to - identifier: unique id for connection - - Returns: None - - """ - log.info(f"On connected {server_type} {client_uri} {identifier}") - connection = Connection( - module_type=server_type, url=client_uri, identifier=identifier - ) - self._connections[identifier] = connection - for c in self._clients: - c.write( - Message(status="connected", connections=[connection]).json().encode() - ) - c.write(b"\n") - - def on_server_disconnected(self, identifier: str) -> None: - """Called when a module has disconnected. - - Args: - identifier: unique id for the connection - - Returns: None - """ - log.info(f"On disconnected {identifier}") - try: - connection = self._connections[identifier] - del self._connections[identifier] - for c in self._clients: - c.write( - Message(status="disconnected", connections=[connection]) - .json() - .encode() - ) - c.write(MessageDelimiter) - except KeyError: - log.exception("Failed to find identifier") - - async def run(self) -> None: - """Run the server.""" - server = await asyncio.start_server( - self._handle_connection, host=self._settings.host, port=self._settings.port - ) - await server.serve_forever() - - async def _handle_connection( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter - ) -> None: - """Handle a client connection to the server.""" - log.info("Client connected to module server.") - - m = Message(status="dump", connections=list(self._connections.values())) - - writer.write(m.json().encode()) - writer.write(MessageDelimiter) - - self._clients.add(writer) - - while True: - if b"" == await reader.read(): - self._clients.remove(writer) - break - - log.info("Client disconnected from module server.") - - -class Connection(BaseModel): - """Model a single module connection.""" - - url: str - module_type: str - identifier: str - - -class Message(BaseModel): - """A message sent to module server clients.""" - - status: Literal["connected", "disconnected", "dump"] - connections: List[Connection] - - -class ModuleServerClient: - """A module server client.""" - - def __init__( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter - ) -> None: - """Constructor.""" - self._reader = reader - self._writer = writer - - @classmethod - async def connect( - cls, - host: str, - port: int, - retries: int = 3, - interval_seconds: float = 0.1, - ) -> ModuleServerClient: - """Connect to the module server. - - Args: - host: module server host. - port: module server port. - retries: number of retries - interval_seconds: time between retries. - - Returns: - None - Raises: - IOError on retry expiry. - """ - r: Optional[asyncio.StreamReader] = None - w: Optional[asyncio.StreamWriter] = None - for i in range(retries): - try: - r, w = await asyncio.open_connection(host=host, port=port) - except OSError: - await asyncio.sleep(interval_seconds) - - if r is not None and w is not None: - return ModuleServerClient(reader=r, writer=w) - else: - raise IOError( - f"Failed to connect to module_server at after {retries} retries." - ) - - async def read(self) -> Message: - """Read a message from the module server.""" - b = await self._reader.readuntil(MessageDelimiter) - m: Message = Message.parse_raw(b) - return m - - def close(self) -> None: - """Close the client.""" - self._writer.close() - - -async def wait_emulators( - client: ModuleServerClient, - modules: Sequence[ModuleType], - timeout: float, -) -> None: - """Wait for module emulators to connect. - - Args: - client: module server client. - modules: collection of of module types to wait for. - timeout: how long to wait for emulators to connect (in seconds) - - Returns: - None - Raises: - asyncio.TimeoutError on timeout. - """ - - async def _wait_modules(cl: ModuleServerClient, module_set: Set[str]) -> None: - """Read messages from module server waiting for modules in module_set to - be connected.""" - while module_set: - m: Message = await cl.read() - if m.status == "dump" or m.status == "connected": - for c in m.connections: - if c.module_type in module_set: - module_set.remove(c.module_type) - elif m.status == "disconnected": - for c in m.connections: - if c.module_type in module_set: - module_set.add(c.module_type) - - log.debug(f"after message: {m}, awaiting module set is: {module_set}") - - await asyncio.wait_for( - _wait_modules(cl=client, module_set=set(n.value for n in modules)), - timeout=timeout, - ) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/__init__.py b/api/src/opentrons/hardware_control/emulation/module_server/__init__.py new file mode 100644 index 00000000000..082295dbfc7 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/__init__.py @@ -0,0 +1,8 @@ +"""Package for the module status server.""" +from .server import ModuleStatusServer +from .client import ModuleStatusClient + +__all__ = [ + "ModuleStatusServer", + "ModuleStatusClient", +] diff --git a/api/src/opentrons/hardware_control/emulation/module_server/client.py b/api/src/opentrons/hardware_control/emulation/module_server/client.py new file mode 100644 index 00000000000..b1051861fc6 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/client.py @@ -0,0 +1,72 @@ +from __future__ import annotations +import asyncio +from asyncio import IncompleteReadError, LimitOverrunError +from typing import Optional + +from opentrons.hardware_control.emulation.module_server.models import Message +from opentrons.hardware_control.emulation.module_server.server import MessageDelimiter + + +class ModuleServerClientError(Exception): + pass + + +class ModuleStatusClient: + """A module server client.""" + + def __init__( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """Constructor.""" + self._reader = reader + self._writer = writer + + @classmethod + async def connect( + cls, + host: str, + port: int, + retries: int = 3, + interval_seconds: float = 0.1, + ) -> ModuleStatusClient: + """Connect to the module server. + + Args: + host: module server host. + port: module server port. + retries: number of retries + interval_seconds: time between retries. + + Returns: + None + Raises: + IOError on retry expiry. + """ + r: Optional[asyncio.StreamReader] = None + w: Optional[asyncio.StreamWriter] = None + for i in range(retries): + try: + r, w = await asyncio.open_connection(host=host, port=port) + break + except OSError: + await asyncio.sleep(interval_seconds) + + if r is not None and w is not None: + return ModuleStatusClient(reader=r, writer=w) + else: + raise IOError( + f"Failed to connect to module_server at after {retries} retries." + ) + + async def read(self) -> Message: + """Read a message from the module server.""" + try: + b = await self._reader.readuntil(MessageDelimiter) + m: Message = Message.parse_raw(b) + return m + except (IncompleteReadError, LimitOverrunError) as e: + raise ModuleServerClientError(str(e)) + + def close(self) -> None: + """Close the client.""" + self._writer.close() diff --git a/api/src/opentrons/hardware_control/emulation/module_server/helpers.py b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py new file mode 100644 index 00000000000..ad4e7ffbabe --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/helpers.py @@ -0,0 +1,129 @@ +"""Utililty methods and classes for interacting with the Module Status Server.""" + +import asyncio +from typing import Sequence, Set, Callable, List, Awaitable + +from opentrons.drivers.rpi_drivers.types import USBPort +from opentrons.hardware_control.emulation.module_server.client import ( + ModuleStatusClient, + ModuleServerClientError, +) +from opentrons.hardware_control.emulation.module_server.models import Message +from opentrons.hardware_control.emulation.module_server.server import log +from opentrons.hardware_control.emulation.settings import Settings +from opentrons.hardware_control.emulation.types import ModuleType +from opentrons.hardware_control.modules import ModuleAtPort + +NotifyMethod = Callable[[List[ModuleAtPort], List[ModuleAtPort]], Awaitable[None]] +"""Signature of method to be notified of new and removed modules.""" + + +async def listen_module_connection(callback: NotifyMethod) -> None: + """Listen for module emulator connections.""" + settings = Settings() + try: + client = await ModuleStatusClient.connect( + host=settings.module_server.host, + port=settings.module_server.port, + interval_seconds=1.0, + ) + listener = ModuleListener(client=client, notify_method=callback) + await listener.run() + except IOError: + log.exception("Failed to connect to module server.") + + +class ModuleListener: + """Provide a callback for listening for new and removed module connections.""" + + def __init__(self, client: ModuleStatusClient, notify_method: NotifyMethod) -> None: + """Constructor. + + Args: + client: A module server client + notify_method: callback method. + + Returns: + None + """ + self._client = client + self._notify_method = notify_method + self._hub_index = 1 + + async def run(self) -> None: + """Run the listener.""" + while True: + try: + m = await self._client.read() + await self.handle_message(message=m) + except ModuleServerClientError: + log.exception("Read error.") + break + + async def handle_message(self, message: Message) -> None: + """Call callback with results of message. + + Args: + message: Message object from module server + + Returns: + None + """ + + def _next_index() -> int: + index = self._hub_index + self._hub_index += 1 + return index + + connections = [ + ModuleAtPort( + port=c.url, + name=c.module_type, + usb_port=USBPort(name=c.identifier, sub_names=[], hub=_next_index()), + ) + for c in message.connections + ] + if message.status == "connected" or message.status == "dump": + await self._notify_method(connections, []) + elif message.status == "disconnected": + await self._notify_method([], connections) + + +async def wait_emulators( + client: ModuleStatusClient, + modules: Sequence[ModuleType], + timeout: float, +) -> None: + """Wait for module emulators to connect. + + Args: + client: module server client. + modules: collection of of module types to wait for. + timeout: how long to wait for emulators to connect (in seconds) + + Returns: + None + Raises: + asyncio.TimeoutError on timeout. + """ + + async def _wait_modules(cl: ModuleStatusClient, module_set: Set[str]) -> None: + """Read messages from module server waiting for modules in module_set to + be connected.""" + while module_set: + m: Message = await cl.read() + if m.status == "dump" or m.status == "connected": + for c in m.connections: + if c.module_type in module_set: + module_set.remove(c.module_type) + elif m.status == "disconnected": + for c in m.connections: + if c.module_type in module_set: + module_set.add(c.module_type) + + log.debug(f"after message: {m}, awaiting module set is: {module_set}") + + await asyncio.wait_for( + _wait_modules(cl=client, module_set=set(n.value for n in modules)), + timeout=timeout, + ) diff --git a/api/src/opentrons/hardware_control/emulation/module_server/models.py b/api/src/opentrons/hardware_control/emulation/module_server/models.py new file mode 100644 index 00000000000..05363bfc109 --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/models.py @@ -0,0 +1,31 @@ +from typing import List + +from typing_extensions import Literal + +from pydantic import BaseModel, Field + + +class ModuleConnection(BaseModel): + """Model a single module connection.""" + + url: str = Field( + ..., + description="The url (port) value the module driver should connect to. " + "For example: socket://host:port", + ) + module_type: str = Field( + ..., description="What kind of module this connection emulates." + ) + identifier: str = Field(..., description="Unique id for this emulator.") + + +class Message(BaseModel): + """A message sent to module server clients.""" + + status: Literal["connected", "disconnected", "dump"] = Field( + ..., + description="`dump` includes a complete list of connected emulators. " + "`connected` for new connections. `disconnected` for emulators " + "that have disconnected. ", + ) + connections: List[ModuleConnection] diff --git a/api/src/opentrons/hardware_control/emulation/module_server/server.py b/api/src/opentrons/hardware_control/emulation/module_server/server.py new file mode 100644 index 00000000000..5a3d696eb7b --- /dev/null +++ b/api/src/opentrons/hardware_control/emulation/module_server/server.py @@ -0,0 +1,108 @@ +"""Server notifying of module connections.""" +import asyncio +import logging +from typing import Dict, Set + +from opentrons.hardware_control.emulation.module_server.models import ( + ModuleConnection, + Message, +) +from opentrons.hardware_control.emulation.proxy import ProxyListener +from opentrons.hardware_control.emulation.settings import ModuleServerSettings +from typing_extensions import Final + +log = logging.getLogger(__name__) + +MessageDelimiter: Final = b"\n" + + +class ModuleStatusServer(ProxyListener): + """The module status server is the emulator equivalent of inotify. A client + can know when an emulated module connects or disconnects. + + Clients connect and read JSON messages (See models module). + """ + + def __init__(self, settings: ModuleServerSettings) -> None: + """Constructor + + Args: + settings: app settings + """ + self._settings = settings + self._connections: Dict[str, ModuleConnection] = {} + self._clients: Set[asyncio.StreamWriter] = set() + + def on_server_connected( + self, server_type: str, client_uri: str, identifier: str + ) -> None: + """Called when a new module has connected. + + Args: + server_type: the type of module + client_uri: the url string for a driver to connect to + identifier: unique id for connection + + Returns: None + + """ + log.info(f"On connected {server_type} {client_uri} {identifier}") + connection = ModuleConnection( + module_type=server_type, url=client_uri, identifier=identifier + ) + self._connections[identifier] = connection + for c in self._clients: + c.write( + Message(status="connected", connections=[connection]).json().encode() + ) + c.write(b"\n") + + def on_server_disconnected(self, identifier: str) -> None: + """Called when a module has disconnected. + + Args: + identifier: unique id for the connection + + Returns: None + """ + log.info(f"On disconnected {identifier}") + try: + connection = self._connections[identifier] + del self._connections[identifier] + for c in self._clients: + c.write( + Message(status="disconnected", connections=[connection]) + .json() + .encode() + ) + c.write(MessageDelimiter) + except KeyError: + log.exception("Failed to find identifier") + + async def run(self) -> None: + """Run the server.""" + server = await asyncio.start_server( + self._handle_connection, host=self._settings.host, port=self._settings.port + ) + await server.serve_forever() + + async def _handle_connection( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """Handle a client connection to the server.""" + log.info("Client connected to module server.") + + # A client connected. Send a dump of all connected modules. + m = Message(status="dump", connections=list(self._connections.values())) + + writer.write(m.json().encode()) + writer.write(MessageDelimiter) + + self._clients.add(writer) + + while True: + if b"" == await reader.read(): + self._clients.remove(writer) + break + + log.info("Client disconnected from module server.") diff --git a/api/src/opentrons/hardware_control/emulation/proxy.py b/api/src/opentrons/hardware_control/emulation/proxy.py index 6877c89ebf5..0dfbb2e2eb6 100644 --- a/api/src/opentrons/hardware_control/emulation/proxy.py +++ b/api/src/opentrons/hardware_control/emulation/proxy.py @@ -1,3 +1,4 @@ +"""The Proxy class module.""" from __future__ import annotations import asyncio import logging @@ -14,7 +15,7 @@ @dataclass(frozen=True) class Connection: - """A connection.""" + """Attributes of a client connected on the server port (module emulator).""" identifier: str reader: asyncio.StreamReader @@ -22,6 +23,9 @@ class Connection: class ProxyListener(ABC): + """Interface defining an object needing to know when a server (module emulator) + connected/disconnected.""" + @abstractmethod def on_server_connected( self, server_type: str, client_uri: str, identifier: str @@ -115,7 +119,7 @@ async def _handle_server_connection( self._cons.append(connection) self._event_listener.on_server_connected( server_type=self._name, - client_uri=f"{socket.gethostname()}:{self._settings.driver_port}", + client_uri=f"socket://{socket.gethostname()}:{self._settings.driver_port}", identifier=connection.identifier, ) diff --git a/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py b/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py index 4c19fe539d2..e05ea632c98 100644 --- a/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py +++ b/api/src/opentrons/hardware_control/emulation/scripts/run_module_emulator.py @@ -17,9 +17,11 @@ from opentrons.hardware_control.emulation.settings import Settings, ProxySettings emulator_builder: Final[Dict[str, Callable[[Settings], AbstractEmulator]]] = { - ModuleType.Magnetic.value: lambda s: MagDeckEmulator(Parser()), - ModuleType.Temperature.value: lambda s: TempDeckEmulator(Parser()), - ModuleType.Thermocycler.value: lambda s: ThermocyclerEmulator(Parser()), + ModuleType.Magnetic.value: lambda s: MagDeckEmulator(Parser(), s.magdeck), + ModuleType.Temperature.value: lambda s: TempDeckEmulator(Parser(), s.tempdeck), + ModuleType.Thermocycler.value: lambda s: ThermocyclerEmulator( + Parser(), s.thermocycler + ), } emulator_port: Final[Dict[str, Callable[[Settings], ProxySettings]]] = { diff --git a/api/src/opentrons/hardware_control/emulation/settings.py b/api/src/opentrons/hardware_control/emulation/settings.py index 8e62cfc1192..d81de05275f 100644 --- a/api/src/opentrons/hardware_control/emulation/settings.py +++ b/api/src/opentrons/hardware_control/emulation/settings.py @@ -1,3 +1,4 @@ +from opentrons.hardware_control.emulation.util import TEMPERATURE_ROOM from pydantic import BaseSettings, BaseModel @@ -17,7 +18,31 @@ class SmoothieSettings(BaseModel): port: int = 9996 -class ProxySettings(BaseSettings): +class BaseModuleSettings(BaseModel): + serial_number: str + model: str + version: str + + +class TemperatureModelSettings(BaseModel): + degrees_per_tick: float = 2.0 + starting: float = float(TEMPERATURE_ROOM) + + +class MagDeckSettings(BaseModuleSettings): + pass + + +class TempDeckSettings(BaseModuleSettings): + temperature: TemperatureModelSettings + + +class ThermocyclerSettings(BaseModuleSettings): + lid_temperature: TemperatureModelSettings + plate_temperature: TemperatureModelSettings + + +class ProxySettings(BaseModel): """Settings for a proxy.""" host: str = "0.0.0.0" @@ -29,13 +54,27 @@ class ModuleServerSettings(BaseModel): """Settings for the module server""" host: str = "0.0.0.0" - port: int = 8888 + port: int = 8989 class Settings(BaseSettings): smoothie: SmoothieSettings = SmoothieSettings() - - host: str = "0.0.0.0" + magdeck: MagDeckSettings = MagDeckSettings( + serial_number="magnetic_emulator", model="mag_deck_v20", version="2.0.0" + ) + tempdeck: TempDeckSettings = TempDeckSettings( + serial_number="temperature_emulator", + model="temp_deck_v20", + version="v2.0.1", + temperature=TemperatureModelSettings(starting=0.0), + ) + thermocycler: ThermocyclerSettings = ThermocyclerSettings( + serial_number="thermocycler_emulator", + model="v02", + version="v1.1.0", + lid_temperature=TemperatureModelSettings(), + plate_temperature=TemperatureModelSettings(), + ) heatershaker_proxy: ProxySettings = ProxySettings( emulator_port=9000, driver_port=9995 diff --git a/api/src/opentrons/hardware_control/emulation/tempdeck.py b/api/src/opentrons/hardware_control/emulation/tempdeck.py index 9e99c1e23a8..22c237ee032 100644 --- a/api/src/opentrons/hardware_control/emulation/tempdeck.py +++ b/api/src/opentrons/hardware_control/emulation/tempdeck.py @@ -9,6 +9,7 @@ from opentrons.drivers.temp_deck.driver import GCODE from opentrons.hardware_control.emulation import util from opentrons.hardware_control.emulation.parser import Parser, Command +from opentrons.hardware_control.emulation.settings import TempDeckSettings from .abstract_emulator import AbstractEmulator from .simulations import Temperature @@ -17,17 +18,15 @@ logger = logging.getLogger(__name__) -SERIAL = "temperature_emulator" -MODEL = "temp_deck_v20" -VERSION = "v2.0.1" - - class TempDeckEmulator(AbstractEmulator): """TempDeck emulator""" - def __init__(self, parser: Parser) -> None: - self.reset() + _temperature: Temperature + + def __init__(self, parser: Parser, settings: TempDeckSettings) -> None: + self._settings = settings self._parser = parser + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -36,7 +35,10 @@ def handle(self, line: str) -> Optional[str]: return None if not joined else joined def reset(self): - self._temperature = Temperature(per_tick=0.25, current=0.0) + self._temperature = Temperature( + per_tick=self._settings.temperature.degrees_per_tick, + current=self._settings.temperature.starting, + ) def _handle(self, command: Command) -> Optional[str]: """Handle a command.""" @@ -57,7 +59,11 @@ def _handle(self, command: Command) -> Optional[str]: elif command.gcode == GCODE.DISENGAGE: self._temperature.deactivate(util.TEMPERATURE_ROOM) elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) elif command.gcode == GCODE.PROGRAMMING_MODE: pass return None diff --git a/api/src/opentrons/hardware_control/emulation/thermocycler.py b/api/src/opentrons/hardware_control/emulation/thermocycler.py index f841b00addd..e35f0d05214 100644 --- a/api/src/opentrons/hardware_control/emulation/thermocycler.py +++ b/api/src/opentrons/hardware_control/emulation/thermocycler.py @@ -8,6 +8,7 @@ from opentrons.drivers.thermocycler.driver import GCODE from opentrons.drivers.types import ThermocyclerLidStatus from opentrons.hardware_control.emulation.parser import Parser, Command +from opentrons.hardware_control.emulation.settings import ThermocyclerSettings from .abstract_emulator import AbstractEmulator from .simulations import Temperature, TemperatureWithHold @@ -16,11 +17,6 @@ logger = logging.getLogger(__name__) -SERIAL = "thermocycler_emulator" -MODEL = "v02" -VERSION = "v1.1.0" - - class ThermocyclerEmulator(AbstractEmulator): """Thermocycler emulator""" @@ -30,9 +26,10 @@ class ThermocyclerEmulator(AbstractEmulator): plate_volume: util.OptionalValue[float] plate_ramp_rate: util.OptionalValue[float] - def __init__(self, parser: Parser) -> None: - self.reset() + def __init__(self, parser: Parser, settings: ThermocyclerSettings) -> None: self._parser = parser + self._settings = settings + self.reset() def handle(self, line: str) -> Optional[str]: """Handle a line""" @@ -41,9 +38,13 @@ def handle(self, line: str) -> Optional[str]: return None if not joined else joined def reset(self): - self._lid_temperature = Temperature(per_tick=2, current=util.TEMPERATURE_ROOM) + self._lid_temperature = Temperature( + per_tick=self._settings.lid_temperature.degrees_per_tick, + current=self._settings.lid_temperature.starting, + ) self._plate_temperature = TemperatureWithHold( - per_tick=2, current=util.TEMPERATURE_ROOM + per_tick=self._settings.plate_temperature.degrees_per_tick, + current=self._settings.plate_temperature.starting, ) self.lid_status = ThermocyclerLidStatus.OPEN self.plate_volume = util.OptionalValue[float]() @@ -115,7 +116,11 @@ def _handle(self, command: Command) -> Optional[str]: # noqa: C901 elif command.gcode == GCODE.DEACTIVATE_BLOCK: self._plate_temperature.deactivate(temperature=util.TEMPERATURE_ROOM) elif command.gcode == GCODE.DEVICE_INFO: - return f"serial:{SERIAL} model:{MODEL} version:{VERSION}" + return ( + f"serial:{self._settings.serial_number} " + f"model:{self._settings.model} " + f"version:{self._settings.version}" + ) return None @staticmethod diff --git a/api/src/opentrons/hardware_control/emulation/types.py b/api/src/opentrons/hardware_control/emulation/types.py index cbdd3eecb11..1a5a7028ef3 100644 --- a/api/src/opentrons/hardware_control/emulation/types.py +++ b/api/src/opentrons/hardware_control/emulation/types.py @@ -4,7 +4,7 @@ class ModuleType(str, Enum): """Module type enumeration.""" - Magnetic = "magnetic" - Temperature = "temperature" + Magnetic = "magdeck" + Temperature = "tempdeck" Thermocycler = "thermocycler" Heatershaker = "heatershaker" diff --git a/api/src/opentrons/hardware_control/module_control.py b/api/src/opentrons/hardware_control/module_control.py index 9b63b70bfc0..f7a5e43eb5e 100644 --- a/api/src/opentrons/hardware_control/module_control.py +++ b/api/src/opentrons/hardware_control/module_control.py @@ -1,16 +1,16 @@ +from __future__ import annotations import logging import asyncio -import os import re from typing import List, Tuple, Optional from glob import glob from opentrons.config import IS_ROBOT, IS_LINUX -from opentrons.drivers.rpi_drivers import types -from opentrons.hardware_control.modules import ModuleAtPort - -from .execution_manager import ExecutionManager -from .types import AionotifyEvent +from opentrons.drivers.rpi_drivers import types, usb, usb_simulator +from opentrons.hardware_control.emulation.module_server.helpers import ( + listen_module_connection, +) +from .types import AionotifyEvent, BoardRevision from . import modules @@ -25,25 +25,32 @@ class AttachedModulesControl: USB port information and finally building a module object. """ - def __init__(self, api): + def __init__(self, api, board_revision: BoardRevision) -> None: self._available_modules: List[modules.AbstractModule] = [] self._api = api + self._usb = ( + usb.USBBus(board_revision) + if not api.is_simulator and IS_ROBOT + else usb_simulator.USBBusSimulator(board_revision) + ) @classmethod - async def build(cls, api_instance): - mc_instance = cls(api_instance) + async def build( + cls, api_instance, board_revision: BoardRevision + ) -> AttachedModulesControl: + mc_instance = cls(api_instance, board_revision) if not api_instance.is_simulator: await mc_instance.register_modules(mc_instance.scan()) + api_instance.loop.create_task( + listen_module_connection(mc_instance.register_modules) + ) + return mc_instance @property def available_modules(self) -> List[modules.AbstractModule]: return self._available_modules - @property - def api(self): - return self._api - async def build_module( self, port: str, @@ -56,9 +63,9 @@ async def build_module( port=port, usb_port=usb_port, which=model, - simulating=self.api.is_simulator, + simulating=self._api.is_simulator, loop=loop, - execution_manager=self.api._execution_manager, + execution_manager=self._api._execution_manager, sim_model=sim_model, ) @@ -107,14 +114,15 @@ async def register_modules( # destroy removed mods await self.unregister_modules(removed_mods_at_ports) - sorted_mods_at_port = self.api._backend._usb.match_virtual_ports( - new_mods_at_ports - ) + sorted_mods_at_port = self._usb.match_virtual_ports(new_mods_at_ports) # build new mods for mod in sorted_mods_at_port: new_instance = await self.build_module( - port=mod.port, usb_port=mod.usb_port, model=mod.name, loop=self.api.loop + port=mod.port, + usb_port=mod.usb_port, + model=mod.name, + loop=self._api.loop, ) self._available_modules.append(new_instance) log.info( @@ -145,7 +153,7 @@ async def parse_modules( for module in self.available_modules: if mod_type == module.name(): matching_modules.append(module) - if self.api.is_simulator: + if self._api.is_simulator: module_builder = { "magdeck": modules.MagDeck.build, "tempdeck": modules.TempDeck.build, @@ -154,10 +162,10 @@ async def parse_modules( if module_builder: simulating_module = await module_builder( port="", - usb_port=self.api._backend._usb.find_port(""), + usb_port=self._usb.find_port(""), simulating=True, - loop=self.api.loop, - execution_manager=ExecutionManager(loop=self.api.loop), + loop=self._api.loop, + execution_manager=self._api._execution_manager, sim_model=by_model.value, ) simulated_module = simulating_module @@ -180,21 +188,6 @@ def scan(self) -> List[modules.ModuleAtPort]: if module_at_port: discovered_modules.append(module_at_port) - # Check for emulator environment variables - emulator_uri = os.environ.get("OT_THERMOCYCLER_EMULATOR_URI") - if emulator_uri: - discovered_modules.append( - ModuleAtPort(port=emulator_uri, name="thermocycler") - ) - - emulator_uri = os.environ.get("OT_TEMPERATURE_EMULATOR_URI") - if emulator_uri: - discovered_modules.append(ModuleAtPort(port=emulator_uri, name="tempdeck")) - - emulator_uri = os.environ.get("OT_MAGNETIC_EMULATOR_URI") - if emulator_uri: - discovered_modules.append(ModuleAtPort(port=emulator_uri, name="magdeck")) - log.debug("Discovered modules: {}".format(discovered_modules)) return discovered_modules @@ -212,14 +205,16 @@ def get_module_at_port(port: str) -> Optional[modules.ModuleAtPort]: return modules.ModuleAtPort(port=f"/dev/{port}", name=name) return None - async def handle_module_appearance(self, event: AionotifyEvent): + async def handle_module_appearance(self, event: AionotifyEvent) -> None: """Only called upon availability of aionotify. Check that the file system has changed and either remove or add modules depending on the result. - :param event_name: The title of the even passed into aionotify. - :param event_flags: AionotifyFlags dataclass that maps flags listed from - the aionotify event. + Args: + event: The event passed from aionotify. + + Returns: + None """ maybe_module_at_port = self.get_module_at_port(event.name) new_modules = None diff --git a/api/src/opentrons/hardware_control/simulator.py b/api/src/opentrons/hardware_control/simulator.py index 8b07b263f5b..684ab0e5bd1 100644 --- a/api/src/opentrons/hardware_control/simulator.py +++ b/api/src/opentrons/hardware_control/simulator.py @@ -14,7 +14,6 @@ from opentrons.drivers.smoothie_drivers import SimulatingDriver from opentrons.drivers.rpi_drivers.gpio_simulator import SimulatingGPIOCharDev -from opentrons.drivers.rpi_drivers import usb_simulator from . import modules from .types import BoardRevision, Axis @@ -154,7 +153,6 @@ def _sanitize_attached_instrument( self._log = MODULE_LOG.getChild(repr(self)) self._strict_attached = bool(strict_attached_instruments) self._board_revision = BoardRevision.OG - self._usb = usb_simulator.USBBusSimulator(self._board_revision) # TODO (lc 05-12-2021) In a follow-up refactor that pulls the execution # manager responsbility into the controller/backend itself as opposed # to the hardware api controller. diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/__init__.py b/api/tests/opentrons/hardware_control/emulation/module_server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py new file mode 100644 index 00000000000..441917b8236 --- /dev/null +++ b/api/tests/opentrons/hardware_control/emulation/module_server/test_helpers.py @@ -0,0 +1,152 @@ +from typing import List + +import pytest +from mock import AsyncMock +from opentrons.drivers.rpi_drivers.types import USBPort +from opentrons.hardware_control.emulation.module_server import ( + helpers, + ModuleStatusClient, +) +from opentrons.hardware_control.emulation.module_server import models +from opentrons.hardware_control.modules import ModuleAtPort + + +@pytest.fixture +def mock_callback() -> AsyncMock: + """Callback mock.""" + return AsyncMock(spec=helpers.NotifyMethod) + + +@pytest.fixture +def mock_client() -> AsyncMock: + """Mock client.""" + return AsyncMock(spec=ModuleStatusClient) + + +@pytest.fixture +def subject(mock_callback: AsyncMock, mock_client: AsyncMock) -> helpers.ModuleListener: + """Test subject.""" + return helpers.ModuleListener(client=mock_client, notify_method=mock_callback) + + +@pytest.fixture +def connections() -> List[models.ModuleConnection]: + """Connection models.""" + return [ + models.ModuleConnection( + url=f"url{i}", module_type=f"module_type{i}", identifier=f"identifier{i}" + ) + for i in range(5) + ] + + +@pytest.fixture +def modules_at_port() -> List[ModuleAtPort]: + """Connection models.""" + return [ + ModuleAtPort( + port=f"url{i}", + name=f"module_type{i}", + usb_port=USBPort(name=f"identifier{i}", sub_names=[], hub=i + 1), + ) + for i in range(5) + ] + + +async def test_handle_message_connected_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=[]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], []) + + +async def test_handle_message_connected_one( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=connections[:1]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with(modules_at_port[:1], []) + + +async def test_handle_message_connected_many( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to add.""" + message = models.Message(status="connected", connections=connections) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with(modules_at_port, []) + + +async def test_handle_message_disconnected_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=[]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], []) + + +async def test_handle_message_disconnected_one( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=connections[:1]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], modules_at_port[:1]) + + +async def test_handle_message_disconnected_many( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to remove.""" + message = models.Message(status="disconnected", connections=connections) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], modules_at_port) + + +async def test_handle_message_dump_empty( + subject: helpers.ModuleListener, mock_callback: AsyncMock +) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=[]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with([], []) + + +async def test_handle_message_dump_one( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=connections[:1]) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with(modules_at_port[:1], []) + + +async def test_handle_message_dump_many( + subject: helpers.ModuleListener, + mock_callback: AsyncMock, + connections: List[models.ModuleConnection], + modules_at_port: List[ModuleAtPort], +) -> None: + """It should call the call back with the correct modules to load.""" + message = models.Message(status="dump", connections=connections) + await subject.handle_message(message=message) + mock_callback.assert_called_once_with(modules_at_port, []) diff --git a/api/tests/opentrons/hardware_control/integration/conftest.py b/api/tests/opentrons/hardware_control/integration/conftest.py index 3ac09b54640..742413da920 100644 --- a/api/tests/opentrons/hardware_control/integration/conftest.py +++ b/api/tests/opentrons/hardware_control/integration/conftest.py @@ -1,8 +1,11 @@ from typing import Iterator +import threading import pytest -import threading import asyncio + +from opentrons.hardware_control.emulation.module_server import ModuleStatusClient +from opentrons.hardware_control.emulation.module_server.helpers import wait_emulators from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control.emulation.types import ModuleType from opentrons.hardware_control.emulation.settings import ( @@ -16,7 +19,6 @@ def emulator_settings() -> Settings: """Emulator settings""" return Settings( - host="0.0.0.0", smoothie=SmoothieSettings( left=PipetteSettings(model="p20_multi_v2.0", id="P3HMV202020041605"), right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), @@ -27,24 +29,32 @@ def emulator_settings() -> Settings: @pytest.fixture(scope="session") def emulation_app(emulator_settings: Settings) -> Iterator[None]: """Run the emulators""" + modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] + + def _run_app() -> None: + asyncio.run(run_app.run(emulator_settings, modules=[m.value for m in modules])) - def runit() -> None: - asyncio.run( - run_app.run( - settings=emulator_settings, - modules=[ - ModuleType.Magnetic, - ModuleType.Temperature, - ModuleType.Thermocycler, - ], - ) + async def _wait_ready() -> None: + c = await ModuleStatusClient.connect( + host="localhost", + port=emulator_settings.module_server.port, + interval_seconds=1, ) + await wait_emulators(client=c, modules=modules, timeout=5) + c.close() - # TODO 20210219 - # The emulators must be run in a separate thread because our serial - # drivers block the main thread. Remove this thread when that is no - # longer true. - t = threading.Thread(target=runit) + def _run_wait_ready() -> None: + asyncio.run(_wait_ready()) + + # Start the emulator thread. + t = threading.Thread(target=_run_app) t.daemon = True t.start() + + # Start the wait for emulator ready thread and wait for it to terminate. + ready_proc = threading.Thread(target=_run_wait_ready) + ready_proc.daemon = True + ready_proc.start() + ready_proc.join() + yield diff --git a/docker-compose.yml b/docker-compose.yml index b66e1846fd4..931d70f49d9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,12 @@ services: build: . command: python3 -m opentrons.hardware_control.emulation.app ports: + - '8888:8888' + - '9000:9000' + - '9002:9002' + - '9003:9003' + - '9004:9004' + - '9995:9995' - '9996:9996' - '9997:9997' - '9998:9998' @@ -17,14 +23,33 @@ services: ports: - '31950:31950' environment: - - OT_API_CONFIG_DIR=/config - - OT_SMOOTHIE_EMULATOR_URI=socket://emulator:9996 - - OT_THERMOCYCLER_EMULATOR_URI=socket://emulator:9997 - - OT_TEMPERATURE_EMULATOR_URI=socket://emulator:9998 - - OT_MAGNETIC_EMULATOR_URI=socket://emulator:9999 + OT_API_CONFIG_DIR: /config + OT_SMOOTHIE_EMULATOR_URI: socket://emulator:9996 + OT_EMULATOR_module_server: '{"host": "emulator"}' links: - 'emulator' depends_on: - 'emulator' volumes: - .opentrons_config:/config:rw + tempdeck: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator tempdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' + thermocycler: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator thermocycler emulator + links: + - 'emulator' + depends_on: + - 'emulator' + magdeck: + build: . + command: python3 -m opentrons.hardware_control.emulation.scripts.run_module_emulator magdeck emulator + links: + - 'emulator' + depends_on: + - 'emulator' diff --git a/g-code-testing/cli.py b/g-code-testing/cli.py index d0481f3b346..8cb73ae1c57 100644 --- a/g-code-testing/cli.py +++ b/g-code-testing/cli.py @@ -78,7 +78,7 @@ def _diff(self) -> str: else: text = "No difference between compared strings" - return text + return text def _configurations(self) -> str: """Get a list of runnable G-Code Configurations""" diff --git a/g-code-testing/g_code_parsing/g_code_engine.py b/g-code-testing/g_code_parsing/g_code_engine.py index 36c323cb617..a1443b85f5d 100644 --- a/g-code-testing/g_code_parsing/g_code_engine.py +++ b/g-code-testing/g_code_parsing/g_code_engine.py @@ -1,8 +1,9 @@ import os import sys import asyncio +import time from multiprocessing import Process -from typing import Generator, Callable +from typing import Generator, Callable, Iterator from collections import namedtuple from opentrons.hardware_control.emulation.settings import Settings @@ -12,7 +13,10 @@ from contextlib import contextmanager from opentrons.protocol_api import ProtocolContext from opentrons.config.robot_configs import build_config -from opentrons.hardware_control.emulation import module_server +from opentrons.hardware_control.emulation.module_server.helpers import ( + wait_emulators, + ModuleStatusClient, +) from opentrons.hardware_control.emulation.scripts import run_app from opentrons.hardware_control import API, ThreadManager from g_code_parsing.g_code_program.g_code_program import ( @@ -45,7 +49,6 @@ class GCodeEngine: def __init__(self, emulator_settings: Settings) -> None: self._config = emulator_settings - self._set_env_vars(emulator_settings) @staticmethod def _get_loop() -> asyncio.AbstractEventLoop: @@ -57,56 +60,52 @@ def _get_loop() -> asyncio.AbstractEventLoop: asyncio.set_event_loop(_loop) return asyncio.get_event_loop() - @staticmethod - def _set_env_vars(settings: Settings) -> None: - """Set URLs of where to find modules and config for smoothie""" - os.environ["OT_MAGNETIC_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.magdeck_proxy.driver_port - ) - os.environ["OT_THERMOCYCLER_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.thermocycler_proxy.driver_port - ) - os.environ["OT_TEMPERATURE_EMULATOR_URI"] = ( - GCodeEngine.URI_TEMPLATE % settings.temperature_proxy.driver_port - ) - - @staticmethod - def _start_emulation_app(emulator_settings: Settings) -> Process: - """Start emulated OT-2""" + @contextmanager + def _emulate(self) -> Iterator[ThreadManager]: + """Context manager that starts emulated OT-2 hardware environment. A + hardware controller is returned.""" modules = [ModuleType.Magnetic, ModuleType.Temperature, ModuleType.Thermocycler] - def runit(): - asyncio.run( - run_app.run(emulator_settings, modules=[m.value for m in modules]) - ) + # Entry point for the emulator app process + def _run_app(): + asyncio.run(run_app.run(self._config, modules=[m.value for m in modules])) - proc = Process(target=runit) + proc = Process(target=_run_app) proc.daemon = True proc.start() + # Entry point for process that waits for emulation to be ready. async def _wait_ready() -> None: - c = await module_server.ModuleServerClient.connect( - host="localhost", port=emulator_settings.module_server.port + c = await ModuleStatusClient.connect( + host="localhost", port=self._config.module_server.port ) - await module_server.wait_emulators(client=c, modules=modules, timeout=5) + await wait_emulators(client=c, modules=modules, timeout=5) c.close() - proc2 = Process(target=lambda: asyncio.run(_wait_ready())) - proc2.start() - proc2.join() + def _run_wait_ready(): + asyncio.run(_wait_ready()) - return proc + ready_proc = Process(target=_run_wait_ready) + ready_proc.daemon = True + ready_proc.start() + ready_proc.join() - @staticmethod - def _emulate_hardware(settings: Settings) -> ThreadManager: - """Created emulated smoothie""" + # Hardware controller conf = build_config({}) emulator = ThreadManager( API.build_hardware_controller, conf, - GCodeEngine.URI_TEMPLATE % settings.smoothie.port, + GCodeEngine.URI_TEMPLATE % self._config.smoothie.port, ) - return emulator + # Wait for modules to be present + while len(emulator.attached_modules) != len(modules): + time.sleep(0.1) + + yield emulator + + # Finished. Stop the emulator + proc.kill() + proc.join() @staticmethod def _get_protocol(file_path: str) -> Protocol: @@ -125,20 +124,16 @@ def run_protocol(self, path: str) -> Generator: :return: GCodeProgram with all the parsed data """ file_path = os.path.join(get_configuration_dir(), path) - app_process = self._start_emulation_app(emulator_settings=self._config) - protocol = self._get_protocol(file_path) - context = ProtocolContext( - implementation=ProtocolContextImplementation( - hardware=self._emulate_hardware(settings=self._config) - ), - loop=self._get_loop(), - ) - parsed_protocol = parse(protocol.text, protocol.filename) - with GCodeWatcher(emulator_settings=self._config) as watcher: - execute.run_protocol(parsed_protocol, context=context) - yield GCodeProgram.from_g_code_watcher(watcher) - app_process.terminate() - app_process.join() + with self._emulate() as h: + protocol = self._get_protocol(file_path) + context = ProtocolContext( + implementation=ProtocolContextImplementation(hardware=h), + loop=self._get_loop(), + ) + parsed_protocol = parse(protocol.text, protocol.filename) + with GCodeWatcher(emulator_settings=self._config) as watcher: + execute.run_protocol(parsed_protocol, context=context) + yield GCodeProgram.from_g_code_watcher(watcher) @contextmanager def run_http(self, executable: Callable): @@ -147,11 +142,7 @@ def run_http(self, executable: Callable): :param executable: Function connected to HTTP Request to execute :return: """ - app_process = self._start_emulation_app(emulator_settings=self._config) - with GCodeWatcher(emulator_settings=self._config) as watcher: - asyncio.run( - executable(hardware=self._emulate_hardware(settings=self._config)) - ) - yield GCodeProgram.from_g_code_watcher(watcher) - app_process.terminate() - app_process.join() + with self._emulate() as h: + with GCodeWatcher(emulator_settings=self._config) as watcher: + asyncio.run(executable(hardware=h)) + yield GCodeProgram.from_g_code_watcher(watcher) diff --git a/g-code-testing/g_code_test_data/http/http_settings.py b/g-code-testing/g_code_test_data/http/http_settings.py index 5847d0fd92a..70ccec3ab12 100644 --- a/g-code-testing/g_code_test_data/http/http_settings.py +++ b/g-code-testing/g_code_test_data/http/http_settings.py @@ -1,11 +1,13 @@ from typing_extensions import Final -from opentrons.hardware_control.emulation.settings import Settings, SmoothieSettings +from opentrons.hardware_control.emulation.settings import ( + Settings, SmoothieSettings, PipetteSettings +) HTTP_SETTINGS = Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_single_v2.1", "id": "P20SV202020070101"} + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101") ) ) diff --git a/g-code-testing/g_code_test_data/http/modules/magdeck.py b/g-code-testing/g_code_test_data/http/modules/magdeck.py index 73248957cbb..d20fae2fa86 100644 --- a/g-code-testing/g_code_test_data/http/modules/magdeck.py +++ b/g-code-testing/g_code_test_data/http/modules/magdeck.py @@ -3,7 +3,7 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.magdeck import SERIAL as SERIAL_NUM + MAGDECK_CALIBRATE = HTTPGCodeConfirmConfig( name='magdeck_calibrate', @@ -11,7 +11,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='calibrate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -22,7 +22,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -33,7 +33,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='engage', args=[5.1]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.magdeck.serial_number, ), settings=HTTP_SETTINGS, ) diff --git a/g-code-testing/g_code_test_data/http/modules/tempdeck.py b/g-code-testing/g_code_test_data/http/modules/tempdeck.py index a67dbea3df9..b3430ca5926 100644 --- a/g-code-testing/g_code_test_data/http/modules/tempdeck.py +++ b/g-code-testing/g_code_test_data/http/modules/tempdeck.py @@ -3,7 +3,7 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.tempdeck import SERIAL as SERIAL_NUM + TEMPDECK_DEACTIVATE = HTTPGCodeConfirmConfig( name='tempdeck_deactivate', @@ -11,7 +11,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -24,7 +24,7 @@ # Keep the args at a low value because the temp starts and 0.0 and only # changes 0.25 degrees every second command=SerialCommand(command_type='set_temperature', args=[2.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) @@ -37,7 +37,7 @@ # This function does not wait on the tempdeck to finish coming to temp # so no need to set to a low value command=SerialCommand(command_type='start_set_temperature', args=[40.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.tempdeck.serial_number, ), settings=HTTP_SETTINGS, ) diff --git a/g-code-testing/g_code_test_data/http/modules/thermocycler.py b/g-code-testing/g_code_test_data/http/modules/thermocycler.py index 646cea1a466..bc02e0d67af 100644 --- a/g-code-testing/g_code_test_data/http/modules/thermocycler.py +++ b/g-code-testing/g_code_test_data/http/modules/thermocycler.py @@ -3,7 +3,6 @@ from g_code_test_data.g_code_configuration import HTTPGCodeConfirmConfig from robot_server.service.legacy.routers.modules import post_serial_command from robot_server.service.legacy.models.modules import SerialCommand -from opentrons.hardware_control.emulation.thermocycler import SERIAL as SERIAL_NUM THERMOCYCLER_CLOSE = HTTPGCodeConfirmConfig( @@ -12,7 +11,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='close'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -23,7 +22,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='open'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -34,7 +33,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -45,7 +44,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate_block'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -56,7 +55,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='deactivate_lid'), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -72,7 +71,7 @@ , 1 ] ), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -83,7 +82,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='set_lid_temperature', args=[37.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) @@ -94,7 +93,7 @@ executable=partial( post_serial_command, command=SerialCommand(command_type='set_temperature', args=[1.0]), - serial=SERIAL_NUM, + serial=HTTP_SETTINGS.thermocycler.serial_number, ), settings=HTTP_SETTINGS, ) diff --git a/g-code-testing/g_code_test_data/protocol/protocol_configurations.py b/g-code-testing/g_code_test_data/protocol/protocol_configurations.py index 4f00cd2b293..96485d03a6c 100644 --- a/g-code-testing/g_code_test_data/protocol/protocol_configurations.py +++ b/g-code-testing/g_code_test_data/protocol/protocol_configurations.py @@ -1,7 +1,8 @@ from typing_extensions import Final -from opentrons.hardware_control.emulation.settings import Settings, SmoothieSettings - +from opentrons.hardware_control.emulation.settings import ( + Settings, SmoothieSettings, PipetteSettings +) from g_code_test_data.g_code_configuration import ProtocolGCodeConfirmConfig import pytest @@ -12,8 +13,8 @@ SWIFT_SMOOTHIE_SETTINGS = Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_multi_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_multi_v2.1", id="P20SV202020070101"), ) ) @@ -32,8 +33,8 @@ s3_path=f"{S3_BASE}/basic_smoothie.txt", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), ) ) ) @@ -44,8 +45,8 @@ s3_path=f"{S3_BASE}/2_single_channel.txt", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p20_single_v2.0", "id": "P20SV202020070101"}, - right={"model": "p300_single_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), + right=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101"), ) ) ) @@ -56,8 +57,8 @@ s3_path=f"{S3_BASE}/2_modules.txt", settings=Settings( smoothie=SmoothieSettings( - left={"model": "p300_single_v2.1", "id": "P20SV202020070101"}, - right={"model": "p20_multi_v2.1", "id": "P20SV202020070101"}, + left=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101"), + right=PipetteSettings(model="p20_multi_v2.1", id="P20SV202020070101"), ) ) ) diff --git a/g-code-testing/tests/g_code_parsing/test_g_code_engine.py b/g-code-testing/tests/g_code_parsing/test_g_code_engine.py index 1a5556628bc..e85f6b423c7 100644 --- a/g-code-testing/tests/g_code_parsing/test_g_code_engine.py +++ b/g-code-testing/tests/g_code_parsing/test_g_code_engine.py @@ -13,7 +13,6 @@ from g_code_parsing.utils import get_configuration_dir CONFIG = Settings( - host="0.0.0.0", smoothie=SmoothieSettings( left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), right=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), From c58e5eb6f84044e3fa5fcd1f351220374bb43256 Mon Sep 17 00:00:00 2001 From: amit lissack Date: Tue, 9 Nov 2021 09:54:32 -0500 Subject: [PATCH 14/16] don't listen for emulators on robot. --- api/src/opentrons/hardware_control/module_control.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/api/src/opentrons/hardware_control/module_control.py b/api/src/opentrons/hardware_control/module_control.py index f7a5e43eb5e..92ec65c1d32 100644 --- a/api/src/opentrons/hardware_control/module_control.py +++ b/api/src/opentrons/hardware_control/module_control.py @@ -40,10 +40,13 @@ async def build( ) -> AttachedModulesControl: mc_instance = cls(api_instance, board_revision) if not api_instance.is_simulator: + # Do an initial scan of modules. await mc_instance.register_modules(mc_instance.scan()) - api_instance.loop.create_task( - listen_module_connection(mc_instance.register_modules) - ) + if not IS_ROBOT: + # Start task that registers emulated modules. + api_instance.loop.create_task( + listen_module_connection(mc_instance.register_modules) + ) return mc_instance From 1f06b7515b43b0ffd298cd89173c7b7ef8e3c772 Mon Sep 17 00:00:00 2001 From: amitlissack Date: Tue, 9 Nov 2021 14:40:47 -0500 Subject: [PATCH 15/16] chore(gcode): reduce time of gcode-testing (#8683) * faster g-code-tests in ci * reduce hold time. * use dev folder in s3 --- .../g_code_test_data/http/http_settings.py | 2 +- .../protocol/protocol_configurations.py | 11 ++++++++--- .../protocol/protocols/2_modules_1s_1m_v2.py | 2 +- .../protocol/protocols/swift_smoke.py | 13 +++++++------ .../protocol/protocols/swift_turbo.py | 16 +++++++++------- 5 files changed, 26 insertions(+), 18 deletions(-) diff --git a/g-code-testing/g_code_test_data/http/http_settings.py b/g-code-testing/g_code_test_data/http/http_settings.py index 70ccec3ab12..ea94dad2990 100644 --- a/g-code-testing/g_code_test_data/http/http_settings.py +++ b/g-code-testing/g_code_test_data/http/http_settings.py @@ -12,5 +12,5 @@ ) -S3_BASE: Final = "http" +S3_BASE: Final = "dev/http" """Base of files in s3""" diff --git a/g-code-testing/g_code_test_data/protocol/protocol_configurations.py b/g-code-testing/g_code_test_data/protocol/protocol_configurations.py index 96485d03a6c..376d1be6fed 100644 --- a/g-code-testing/g_code_test_data/protocol/protocol_configurations.py +++ b/g-code-testing/g_code_test_data/protocol/protocol_configurations.py @@ -15,11 +15,16 @@ smoothie=SmoothieSettings( left=PipetteSettings(model="p20_single_v2.0", id="P20SV202020070101"), right=PipetteSettings(model="p300_multi_v2.1", id="P20SV202020070101"), - ) + ), ) +# Set up the temperature ramp. +SWIFT_SMOOTHIE_SETTINGS.thermocycler.lid_temperature.degrees_per_tick = 50 +SWIFT_SMOOTHIE_SETTINGS.thermocycler.plate_temperature.degrees_per_tick = 50 +SWIFT_SMOOTHIE_SETTINGS.tempdeck.temperature.degrees_per_tick = 50 + -S3_BASE: Final = "protocol" +S3_BASE: Final = "dev/protocol" """Base path of files in s3.""" ################## @@ -59,7 +64,7 @@ smoothie=SmoothieSettings( left=PipetteSettings(model="p300_single_v2.1", id="P20SV202020070101"), right=PipetteSettings(model="p20_multi_v2.1", id="P20SV202020070101"), - ) + ), ) ) diff --git a/g-code-testing/g_code_test_data/protocol/protocols/2_modules_1s_1m_v2.py b/g-code-testing/g_code_test_data/protocol/protocols/2_modules_1s_1m_v2.py index 7ac83f9a522..a04afebfce3 100644 --- a/g-code-testing/g_code_test_data/protocol/protocols/2_modules_1s_1m_v2.py +++ b/g-code-testing/g_code_test_data/protocol/protocols/2_modules_1s_1m_v2.py @@ -31,7 +31,7 @@ def run(ctx): # Magnetic Module Testing magdeck.engage(height=10) - ctx.delay(seconds=30) + ctx.delay(seconds=1) magdeck.disengage() ctx.comment(f"mag status {magdeck.status}") magdeck.engage() diff --git a/g-code-testing/g_code_test_data/protocol/protocols/swift_smoke.py b/g-code-testing/g_code_test_data/protocol/protocols/swift_smoke.py index b076648e629..f1515bd67ce 100644 --- a/g-code-testing/g_code_test_data/protocol/protocols/swift_smoke.py +++ b/g-code-testing/g_code_test_data/protocol/protocols/swift_smoke.py @@ -82,8 +82,9 @@ def run(protocol_context): # Run Enzymatic Prep Profile thermocycler.close_lid() thermocycler.set_block_temperature(70) - thermocycler.set_block_temperature(32, hold_time_minutes=1) - thermocycler.set_block_temperature(64.5, hold_time_minutes=1) + # Changed from hold_time_minutes to hold_time_seconds for faster running in CI + thermocycler.set_block_temperature(32, hold_time_seconds=1) + thermocycler.set_block_temperature(64.5, hold_time_seconds=1) thermocycler.set_block_temperature(4) thermocycler.deactivate_lid() thermocycler.open_lid() @@ -271,10 +272,10 @@ def run(protocol_context): COVER_TEMP = 105 PLATE_TEMP_PRE = 4 - PLATE_TEMP_HOLD_1 = (97, 5) # 30) - PLATE_TEMP_HOLD_2 = (97, 5) # 10) - PLATE_TEMP_HOLD_3 = (59.5, 5) # 30) - PLATE_TEMP_HOLD_4 = (67.3, 5) # 30) + PLATE_TEMP_HOLD_1 = (97, 1) # 30) + PLATE_TEMP_HOLD_2 = (97, 1) # 10) + PLATE_TEMP_HOLD_3 = (59.5, 1) # 30) + PLATE_TEMP_HOLD_4 = (67.3, 1) # 30) # PLATE_TEMP_HOLD_5 = (72, 300) PLATE_TEMP_POST = 4 NUM_CYCLES = 5 diff --git a/g-code-testing/g_code_test_data/protocol/protocols/swift_turbo.py b/g-code-testing/g_code_test_data/protocol/protocols/swift_turbo.py index ccc1d8c18fa..2d8cc1d8747 100644 --- a/g-code-testing/g_code_test_data/protocol/protocols/swift_turbo.py +++ b/g-code-testing/g_code_test_data/protocol/protocols/swift_turbo.py @@ -83,8 +83,9 @@ def run(protocol_context): # Run Enzymatic Prep Profile thermocycler.close_lid() thermocycler.set_lid_temperature(70) - thermocycler.set_block_temperature(32, hold_time_minutes=1) - thermocycler.set_block_temperature(64.5, hold_time_minutes=1) + # Changed from hold_time_minutes to hold_time_seconds for faster running in CI + thermocycler.set_block_temperature(32, hold_time_seconds=1) + thermocycler.set_block_temperature(64.5, hold_time_seconds=1) thermocycler.set_block_temperature(4) thermocycler.deactivate_lid() thermocycler.open_lid() @@ -110,7 +111,8 @@ def run(protocol_context): p20.blow_out(well.top(-7)) p20.drop_tip() - thermocycler.set_block_temperature(20.2, hold_time_minutes=2) + # Changed from hold_time_minutes to hold_time_seconds for faster running in CI + thermocycler.set_block_temperature(20.2, hold_time_seconds=1) thermocycler.set_block_temperature(4) """Ligation Purification""" @@ -272,10 +274,10 @@ def run(protocol_context): COVER_TEMP = 105 PLATE_TEMP_PRE = 4 - PLATE_TEMP_HOLD_1 = (97, 2) # 30) - PLATE_TEMP_HOLD_2 = (97, 2) # 10) - PLATE_TEMP_HOLD_3 = (59.5, 2) # 30) - PLATE_TEMP_HOLD_4 = (67.3, 2) # 30) + PLATE_TEMP_HOLD_1 = (97, 1) # 30) + PLATE_TEMP_HOLD_2 = (97, 1) # 10) + PLATE_TEMP_HOLD_3 = (59.5, 1) # 30) + PLATE_TEMP_HOLD_4 = (67.3, 1) # 30) # PLATE_TEMP_HOLD_5 = (72, 300) PLATE_TEMP_POST = 4 NUM_CYCLES = 5 From 5b2191bbbade2297538c91af41ccde1925380cec Mon Sep 17 00:00:00 2001 From: amit lissack Date: Tue, 9 Nov 2021 16:47:49 -0500 Subject: [PATCH 16/16] wait. --- .../hardware_control/emulation/test_proxy.py | 66 +++++++++++++++---- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/api/tests/opentrons/hardware_control/emulation/test_proxy.py b/api/tests/opentrons/hardware_control/emulation/test_proxy.py index bc128f00e86..5c61006cf7b 100644 --- a/api/tests/opentrons/hardware_control/emulation/test_proxy.py +++ b/api/tests/opentrons/hardware_control/emulation/test_proxy.py @@ -2,9 +2,12 @@ from typing import AsyncIterator import pytest -from mock import MagicMock -from opentrons.hardware_control.emulation.proxy import Proxy, ProxySettings +from opentrons.hardware_control.emulation.proxy import ( + Proxy, + ProxySettings, + ProxyListener, +) @pytest.fixture @@ -13,13 +16,37 @@ def setting() -> ProxySettings: return ProxySettings(emulator_port=12345, driver_port=12346) +class SimpleProxyListener(ProxyListener): + def __init__(self) -> None: + self._count = 0 + + def on_server_connected( + self, server_type: str, client_uri: str, identifier: str + ) -> None: + self._count += 1 + + def on_server_disconnected(self, identifier: str) -> None: + self._count -= 1 + + async def wait_count(self, count: int) -> None: + while count != self._count: + await asyncio.sleep(0.01) + + +@pytest.fixture +def proxy_listener() -> SimpleProxyListener: + """A proxy listener.""" + return SimpleProxyListener() + + @pytest.fixture async def subject( - loop: asyncio.AbstractEventLoop, setting: ProxySettings + loop: asyncio.AbstractEventLoop, + setting: ProxySettings, + proxy_listener: SimpleProxyListener, ) -> AsyncIterator[Proxy]: """Test subject.""" - mock_listener = MagicMock() - p = Proxy("proxy", mock_listener, setting) + p = Proxy("proxy", proxy_listener, setting) task = loop.create_task(p.run()) yield p task.cancel() @@ -29,22 +56,28 @@ async def subject( pass -async def test_driver_route_message(subject: Proxy, setting: ProxySettings) -> None: +async def test_driver_route_message( + subject: Proxy, setting: ProxySettings, proxy_listener: SimpleProxyListener +) -> None: """It should route a message to an emulator.""" emulator = await asyncio.open_connection( host="localhost", port=setting.emulator_port ) + await proxy_listener.wait_count(1) driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) driver[1].write(b"abc") r = await emulator[0].read(3) assert r == b"abc" -async def test_emulator_route_message(subject: Proxy, setting: ProxySettings) -> None: +async def test_emulator_route_message( + subject: Proxy, setting: ProxySettings, proxy_listener: SimpleProxyListener +) -> None: """It should route a message to a driver.""" emulator = await asyncio.open_connection( host="localhost", port=setting.emulator_port ) + await proxy_listener.wait_count(1) driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) emulator[1].write(b"abc") r = await driver[0].read(3) @@ -52,7 +85,7 @@ async def test_emulator_route_message(subject: Proxy, setting: ProxySettings) -> async def test_driver_route_message_two_connections( - subject: Proxy, setting: ProxySettings + subject: Proxy, setting: ProxySettings, proxy_listener: SimpleProxyListener ) -> None: """It should route messages to correct emulator.""" emulator1 = await asyncio.open_connection( @@ -61,6 +94,7 @@ async def test_driver_route_message_two_connections( emulator2 = await asyncio.open_connection( host="localhost", port=setting.emulator_port ) + await proxy_listener.wait_count(2) driver1 = await asyncio.open_connection(host="localhost", port=setting.driver_port) driver2 = await asyncio.open_connection(host="localhost", port=setting.driver_port) driver1[1].write(b"abc") @@ -72,7 +106,7 @@ async def test_driver_route_message_two_connections( async def test_emulator_route_message_two_connections( - subject: Proxy, setting: ProxySettings + subject: Proxy, setting: ProxySettings, proxy_listener: SimpleProxyListener ) -> None: """It should route messages to correct driver.""" emulator1 = await asyncio.open_connection( @@ -81,6 +115,7 @@ async def test_emulator_route_message_two_connections( emulator2 = await asyncio.open_connection( host="localhost", port=setting.emulator_port ) + await proxy_listener.wait_count(2) driver1 = await asyncio.open_connection(host="localhost", port=setting.driver_port) driver2 = await asyncio.open_connection(host="localhost", port=setting.driver_port) emulator1[1].write(b"abc") @@ -98,12 +133,13 @@ async def test_driver_and_no_emulator(subject: Proxy, setting: ProxySettings) -> async def test_two_driver_and_one_emulator( - subject: Proxy, setting: ProxySettings + subject: Proxy, setting: ProxySettings, proxy_listener: SimpleProxyListener ) -> None: """It should fail to read if no emulator.""" emulator = await asyncio.open_connection( host="localhost", port=setting.emulator_port ) + await proxy_listener.wait_count(1) driver1 = await asyncio.open_connection(host="localhost", port=setting.driver_port) driver2 = await asyncio.open_connection(host="localhost", port=setting.driver_port) emulator[1].write(b"abc") @@ -111,11 +147,14 @@ async def test_two_driver_and_one_emulator( assert b"" == await driver2[0].read(n=3) -async def test_driver_reconnect(subject: Proxy, setting: ProxySettings) -> None: +async def test_driver_reconnect( + subject: Proxy, setting: ProxySettings, proxy_listener: SimpleProxyListener +) -> None: """It should allow a second driver to claim a formerly used emulator.""" emulator = await asyncio.open_connection( host="localhost", port=setting.emulator_port ) + await proxy_listener.wait_count(1) driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) emulator[1].write(b"abc") assert b"abc" == await driver[0].read(n=3) @@ -127,11 +166,14 @@ async def test_driver_reconnect(subject: Proxy, setting: ProxySettings) -> None: assert b"abc" == await driver[0].read(n=3) -async def test_emulator_disconnects(subject: Proxy, setting: ProxySettings) -> None: +async def test_emulator_disconnects( + subject: Proxy, setting: ProxySettings, proxy_listener: SimpleProxyListener +) -> None: """It should disconnect driver when emulator disconnects.""" emulator = await asyncio.open_connection( host="localhost", port=setting.emulator_port ) + await proxy_listener.wait_count(1) driver = await asyncio.open_connection(host="localhost", port=setting.driver_port) emulator[1].close()