feat(api): add module firmware update endpoint

Closes #1654

api/opentrons/config/modules/95-opentrons-modules.rules

@@ -1,2 +1,3 @@
-KERNEL=="ttyACM[0-9]*" SUBSYSTEMS=="usb", ATTRS{idProduct}=="ee93", ATTRS{idVendor}=="04d8", SYMLINK+="modules/tty%n_tempdeck"
-KERNEL=="ttyACM[0-9]*" SUBSYSTEMS=="usb", ATTRS{idProduct}=="ee90", ATTRS{idVendor}=="04d8", SYMLINK+="modules/tty%n_magdeck"
+KERNEL=="ttyACM[0-9]*", SUBSYSTEMS=="usb", ATTRS{idProduct}=="ee93", ATTRS{idVendor}=="04d8", SYMLINK+="modules/tty%n_tempdeck"
+KERNEL=="ttyACM[0-9]*", SUBSYSTEMS=="usb", ATTRS{idProduct}=="ee90", ATTRS{idVendor}=="04d8", SYMLINK+="modules/tty%n_magdeck"
+KERNEL=="ttyACM[0-9]*", SUBSYSTEMS=="usb", ATTRS{idProduct}=="ee5a", ATTRS{idVendor}=="04d8", SYMLINK+="modules/tty%n_bootloader"

api/opentrons/drivers/mag_deck/driver.py

@@ -158,7 +158,7 @@ def __init__(self, config={}):
 
     def connect(self, port=None) -> str:
         '''
-        :param port: '/dev/ttyMagDeck'
+        :param port: '/dev/modules/ttyn_magdeck'
         NOTE: Using the symlink above to connect makes sure that the robot
         connects/reconnects to the module even after a device
         reset/reconnection

api/opentrons/modules/__init__.py

@@ -1,14 +1,22 @@
 import os
 import logging
 import re
+import asyncio
+import time
 from opentrons.modules.magdeck import MagDeck
 from opentrons.modules.tempdeck import TempDeck
 from opentrons import robot, labware
 
 log = logging.getLogger(__name__)
 
+PORT_SEARCH_TIMEOUT = 5.5
 SUPPORTED_MODULES = {'magdeck': MagDeck, 'tempdeck': TempDeck}
 
+# avrdude_options
+PART_NO = 'atmega32u4'
+PROGRAMMER_ID = 'avr109'
+BAUDRATE = '57600'
+
 
 class UnsupportedModuleError(Exception):
     pass
@@ -76,3 +84,128 @@ def discover_and_connect():
             log.exception('Failed to connect module')
 
     return discovered_modules
+
+
+async def enter_bootloader(module):
+    """
+    Using the driver method, enter bootloader mode of the atmega32u4.
+    The bootloader mode opens a new port on the uC to upload the hex file.
+    After receiving a 'dfu' command, the firmware provides a 3-second window to
+    close the current port so as to do a clean switch to the bootloader port.
+    The new port shows up as 'ttyn_bootloader' on the pi; upload fw through it.
+    NOTE: Modules with old bootloader will have the bootloader port show up as
+    a regular module port- 'ttyn_tempdeck'/ 'ttyn_magdeck' with the port number
+    being either different or same as the one that the module was originally on
+    So we check for changes in ports and use the appropriate one
+    """
+    # Required for old bootloader
+    ports_before_dfu_mode = await _discover_ports()
+
+    module._driver.enter_programming_mode()
+    module.disconnect()
+    new_port = ''
+    then = time.time()
+    while time.time() - then < PORT_SEARCH_TIMEOUT:
+        new_port = await _port_poll(
+            _has_old_bootloader(module), ports_before_dfu_mode)
+        if new_port:
+            log.debug("Found new (bootloader) port: {}".format(new_port))
+            break
+        else:
+            await asyncio.sleep(0.05)
+    return new_port
+
+
+async def update_firmware(module, firmware_file_path, config_file_path, loop):
+    """
+    Run avrdude firmware upload command. Switch back to normal module port
+
+    Note: For modules with old bootloader, the kernel could assign the module
+    a new port after the update (since the board is automatically reset).
+    Scan for such a port change and use the appropriate port
+    """
+    # TODO: Make sure the module isn't in the middle of operation
+
+    ports_before_update = await _discover_ports()
+
+    proc = await asyncio.create_subprocess_exec(
+        'avrdude', '-C{}'.format(config_file_path), '-v',
+        '-p{}'.format(PART_NO),
+        '-c{}'.format(PROGRAMMER_ID),
+        '-P{}'.format(module.port),
+        '-b{}'.format(BAUDRATE), '-D',
+        '-Uflash:w:{}:i'.format(firmware_file_path),
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE, loop=loop)
+    await proc.wait()
+
+    _result = await proc.communicate()
+    result = _result[1].decode()
+    log.debug(result)
+    log.debug("Switching back to non-bootloader port")
+    module._port = _port_on_mode_switch(ports_before_update)
+
+    return _format_avrdude_response(result)
+
+
+def _format_avrdude_response(raw_response):
+    response_msg = {'result': '', 'avrdude_response': ''}
+    avrdude_log = ''
+    for line in raw_response.splitlines():
+        if 'avrdude:' in line and line != raw_response.splitlines()[1]:
+            avrdude_log += line.lstrip('avrdude:') + '..'
+            if 'flash verified' in line:
+                response_msg['result'] = 'Firmware update successful'
+                response_msg['avrdude_response'] = line.lstrip('avrdude: ')
+    if not response_msg['result']:
+        response_msg['result'] = 'Firmware update failed'
+        response_msg['avrdude_response'] = avrdude_log
+    return response_msg
+
+
+async def _port_on_mode_switch(ports_before_switch):
+    ports_after_switch = await _discover_ports()
+    new_port = ''
+    if len(ports_after_switch) >= len(ports_before_switch) and \
+            not set(ports_before_switch) == set(ports_after_switch):
+        new_ports = list(filter(
+            lambda x: x not in ports_before_switch,
+            ports_after_switch))
+        if len(new_ports) > 1:
+            raise OSError('Multiple new ports found on mode switch')
+        new_port = '/dev/modules/{}'.format(new_ports[0])
+    return new_port
+
+
+async def _port_poll(is_old_bootloader, ports_before_switch=None):
+    """
+    Checks for the bootloader port
+    """
+    new_port = ''
+    if is_old_bootloader:
+        new_port = await _port_on_mode_switch(ports_before_switch)
+    else:
+        ports = await _discover_ports()
+        discovered_ports = list(filter(
+            lambda x: x.endswith('bootloader'), ports))
+        if len(discovered_ports) == 1:
+            new_port = '/dev/modules/{}'.format(discovered_ports[0])
+    return new_port
+
+
+def _has_old_bootloader(module):
+    return True if module.device_info.get('model') == 'temp_deck_v1' or \
+                   module.device_info.get('model') == 'temp_deck_v2' else False
+
+
+async def _discover_ports():
+    if os.environ.get('RUNNING_ON_PI') and os.path.isdir('/dev/modules'):
+        for attempt in range(2):
+            # Measure for race condition where port is being switched in
+            # between calls to isdir() and listdir()
+            try:
+                return os.listdir('/dev/modules')
+            except (FileNotFoundError, OSError):
+                pass
+            await asyncio.sleep(2)
+        raise Exception("No /dev/modules found. Try again")

api/opentrons/server/endpoints/update.py

@@ -2,10 +2,14 @@
 import asyncio
 import shutil
 import os
-
+import opentrons
+import tempfile
+from aiohttp import web
 from opentrons import robot
+from opentrons import modules
 
 log = logging.getLogger(__name__)
+UPDATE_TIMEOUT = 15
 
 
 def _ensure_programmer_executable():
@@ -68,3 +72,88 @@ async def _update_firmware(filename, loop, explicit_modeset=True):
     robot._driver._setup()
 
     return res
+
+
+async def update_module_firmware(request):
+    """
+     This handler accepts a POST request with Content-Type: multipart/form-data
+     and a file field in the body named "module_firmware". The file should
+     be a valid HEX image to be flashed to the atmega32u4. The received file is
+     sent via USB to the board and flashed by the avr109 bootloader. The file
+     is then deleted and a success code is returned
+    """
+    log.debug('Update Firmware request received')
+    data = await request.post()
+    module_serial = request.match_info['serial']
+
+    res = await _update_module_firmware(module_serial,
+                                        data['module_firmware'],
+                                        request.loop)
+    if 'successful' not in res['message']['result']:
+        if 'avrdude_response' in res['message'] and \
+                'checksum mismatch' in res['message']['avrdude_response']:
+                status = 400
+        elif 'not found' in res['message']['result']:
+            status = 404
+        else:
+            status = 500
+        log.error(res)
+    else:
+        status = 200
+        log.info(res)
+    return web.json_response(res, status=status)
+
+
+async def _update_module_firmware(module_serial, data, loop=None):
+
+    fw_filename = data.filename
+    content = data.file.read()
+    log.info('Preparing to flash firmware image {}'.format(fw_filename))
+    config_file_path = os.path.join(opentrons.HERE,
+                                    'config', 'modules', 'avrdude.conf')
+    with tempfile.NamedTemporaryFile(suffix=fw_filename) as fp:
+        fp.write(content)
+        # returns a dict of 'result' & 'avrdude_response'
+        msg = await _upload_to_module(module_serial, fp.name,
+                                      config_file_path, loop=loop)
+    log.info('Firmware update complete')
+    res = {'message': msg, 'filename': fw_filename}
+    return res
+
+
+async def _upload_to_module(serialnum, fw_filename, config_file_path, loop):
+    """
+    This method remains in the API currently because of its use of the robot
+    singleton's copy of the api object & driver. This should move to the server
+    lib project eventually and use its own driver object (preferably involving
+    moving the drivers themselves to the serverlib)
+    """
+
+    # ensure there is a reference to the port
+    if not robot.is_connected():
+        robot.connect()
+    for module in robot.modules:
+        module.disconnect()
+    robot.modules = modules.discover_and_connect()
+    res = ''
+    for module in robot.modules:
+        if module.device_info.get('serial') == serialnum:
+            log.info("Module with serial {} found".format(serialnum))
+            bootloader_port = await modules.enter_bootloader(module)
+            if bootloader_port:
+                module._port = bootloader_port
+            # else assume old bootloader connection on existing module port
+            log.info("Uploading file to port:{} using config file {}".format(
+                module.port, config_file_path))
+            log.info("Flashing firmware. This will take a few seconds")
+            try:
+                res = await asyncio.wait_for(
+                    modules.update_firmware(
+                        module, fw_filename, config_file_path, loop),
+                    UPDATE_TIMEOUT)
+            except asyncio.TimeoutError:
+                return {'result': 'AVRDUDE not responding'}
+            break
+    if not res:
+        res = {'result': 'Module {} not found'.format(serialnum)}
+    return res

api/opentrons/server/main.py

@@ -11,7 +11,7 @@
 from opentrons.api import MainRouter
 from opentrons.server.rpc import Server
 from opentrons.server import endpoints as endp
-from opentrons.server.endpoints import (wifi, control, settings)
+from opentrons.server.endpoints import (wifi, control, settings, update)
 from opentrons.config import feature_flags as ff
 from opentrons.util import environment
 from opentrons.deck_calibration import endpoints as dc_endp
@@ -185,6 +185,8 @@ def init(loop=None):
         '/server/update', endpoints.update_api)
     server.app.router.add_post(
         '/server/update/firmware', endpoints.update_firmware)
+    server.app.router.add_post(
+        '/modules/{serial}/update', update.update_module_firmware)
     server.app.router.add_get(
         '/server/update/ignore', endpoints.get_ignore_version)
     server.app.router.add_post(

api/tests/opentrons/labware/test_modules.py

@@ -93,3 +93,91 @@ def mock_get_info(self):
     robot.modules = [tempdeck]
     modules.load('tempdeck', '5')
     assert connected
+    tempdeck.disconnect()  # Necessary to kill the thread started by connect()
+
+
+@pytest.fixture
+def old_bootloader_module():
+    module = modules.TempDeck(port='/dev/modules/tty0_tempdeck')
+    module._device_info = {'model': 'temp_deck_v1'}
+    module._driver = TempDeckDriver()
+    return module
+
+
+@pytest.fixture
+def new_bootloader_module():
+    module = modules.TempDeck(port='/dev/modules/tty0_tempdeck')
+    module._device_info = {'model': 'temp_deck_v1.1'}
+    module._driver = TempDeckDriver()
+    return module
+
+
+async def test_enter_bootloader(
+        new_bootloader_module, virtual_smoothie_env, monkeypatch):
+
+    async def mock_discover_ports_before_dfu_mode():
+        return 'tty0_tempdeck'
+
+    def mock_enter_programming_mode(self):
+        return 'ok\n\rok\n\r'
+
+    async def mock_port_poll(_has_old_bootloader, ports_before_dfu_mode):
+        return '/dev/modules/tty0_bootloader'
+
+    monkeypatch.setattr(
+        TempDeckDriver, 'enter_programming_mode', mock_enter_programming_mode)
+    monkeypatch.setattr(
+        modules, '_discover_ports', mock_discover_ports_before_dfu_mode)
+    monkeypatch.setattr(modules, '_port_poll', mock_port_poll)
+
+    bootloader_port = await modules.enter_bootloader(new_bootloader_module)
+    assert bootloader_port == '/dev/modules/tty0_bootloader'
+
+
+def test_old_bootloader_check(
+        old_bootloader_module, new_bootloader_module, virtual_smoothie_env):
+    assert modules._has_old_bootloader(old_bootloader_module)
+    assert not modules._has_old_bootloader(new_bootloader_module)
+
+
+async def test_port_poll(virtual_smoothie_env, monkeypatch):
+    has_old_bootloader = False
+
+    # Case 1: Bootloader port is successfully opened on the module
+    async def mock_discover_ports1():
+        return ['tty0_magdeck', 'tty1_bootloader']
+    monkeypatch.setattr(modules, '_discover_ports', mock_discover_ports1)
+
+    port_found = await modules._port_poll(has_old_bootloader, None)
+    assert port_found == '/dev/modules/tty1_bootloader'
+
+    # Case 2: Switching to bootloader mode failed
+    async def mock_discover_ports2():
+        return ['tty0_magdeck', 'tty1_tempdeck']
+    monkeypatch.setattr(modules, '_discover_ports', mock_discover_ports2)
+
+    port_found = await modules._port_poll(has_old_bootloader, None)
+    assert not port_found
+
+
+async def test_old_bootloader_port_poll(virtual_smoothie_env, monkeypatch):
+    ports_before_switch = ['tty0_magdeck', 'tty1_tempdeck']
+    has_old_bootloader = True
+
+    # Case 1: Bootloader is opened on same port
+    async def mock_discover_ports():
+        return ['tty0_magdeck', 'tty1_tempdeck']
+    monkeypatch.setattr(modules, '_discover_ports', mock_discover_ports)
+
+    port_found = await modules._port_poll(has_old_bootloader,
+                                          ports_before_switch)
+    assert not port_found
+
+    # Case 2: Bootloader is opened on a different port
+    async def mock_discover_ports():
+        return ['tty2_magdeck', 'tty1_tempdeck']
+    monkeypatch.setattr(modules, '_discover_ports', mock_discover_ports)
+
+    port_found = await modules._port_poll(has_old_bootloader,
+                                          ports_before_switch)
+    assert port_found == '/dev/modules/tty2_magdeck'