From 1b18003930de3230f4dd71143f60bb350be49fb9 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Sun, 26 Nov 2017 21:18:53 -0500 Subject: [PATCH 01/12] wip - First step towards tests of serial attached devices, but with fake devices. --- pocs/tests/serial_handlers/__init__.py | 0 pocs/tests/serial_handlers/protocol_test.py | 167 ++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 pocs/tests/serial_handlers/__init__.py create mode 100644 pocs/tests/serial_handlers/protocol_test.py diff --git a/pocs/tests/serial_handlers/__init__.py b/pocs/tests/serial_handlers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pocs/tests/serial_handlers/protocol_test.py b/pocs/tests/serial_handlers/protocol_test.py new file mode 100644 index 000000000..bd2125199 --- /dev/null +++ b/pocs/tests/serial_handlers/protocol_test.py @@ -0,0 +1,167 @@ +# This is based on PySerial's file test/handlers/protocol_test.py (i.e. is +# a copy of it, that I'll then modify for testing). +# +# This module implements a URL dummy handler for serial_for_url. +# Apparently the file name needs to be protocol_, +# where here that is "test", allowing URL's of the format "test://". + +from serial.serialutil import SerialBase, SerialException, portNotOpenError + +import io +import logging +import socket +import time + +# map log level names to constants. used in fromURL() +LOGGER_LEVELS = { + 'debug': logging.DEBUG, + 'info': logging.INFO, + 'warning': logging.WARNING, + 'error': logging.ERROR, + } + +class Serial(SerialBase, io.RawIOBase): + """Serial port implementation for plain sockets.""" + + def open(self): + """Open port with current settings. This may throw a SerialException + if the port cannot be opened.""" + self.logger = None + if self._port is None: + raise SerialException("Port must be configured before it can be used.") + # not that there anything to configure... + self._reconfigurePort() + # all things set up get, now a clean start + self._isOpen = True + + def _reconfigurePort(self): + """Set communication parameters on opened port. for the test:// + protocol all settings are ignored!""" + if self.logger: + self.logger.info('ignored port configuration change') + + def close(self): + """Close port""" + if self._isOpen: + self._isOpen = False + + def makeDeviceName(self, port): + raise SerialException("there is no sensible way to turn numbers into URLs") + + def fromURL(self, url): + """extract host and port from an URL string""" + if url.lower().startswith("test://"): url = url[7:] + try: + # is there a "path" (our options)? + if '/' in url: + # cut away options + url, options = url.split('/', 1) + # process options now, directly altering self + for option in options.split('/'): + if '=' in option: + option, value = option.split('=', 1) + else: + value = None + if option == 'logging': + logging.basicConfig() # XXX is that good to call it here? + self.logger = logging.getLogger('pySerial.test') + self.logger.setLevel(LOGGER_LEVELS[value]) + self.logger.debug('enabled logging') + else: + raise ValueError('unknown option: {!r}'.format(option)) + except ValueError as e: + raise SerialException('expected a string in the form "[test://][option[/option...]]": {}'.format(e)) + return (host, port) + + # - - - - - - - - - - - - - - - - - - - - - - - - + + def inWaiting(self): + """Return the number of characters currently in the input buffer.""" + if not self._isOpen: raise portNotOpenError + if self.logger: + # set this one to debug as the function could be called often... + self.logger.debug('WARNING: inWaiting returns dummy value') + return 0 # hmmm, see comment in read() + + def read(self, size=1): + """Read size bytes from the serial port. If a timeout is set it may + return less characters as requested. With no timeout it will block + until the requested number of bytes is read.""" + if not self._isOpen: raise portNotOpenError + data = '123' # dummy data + return bytes(data) + + def write(self, data): + """Output the given string over the serial port. Can block if the + connection is blocked. May raise SerialException if the connection is + closed.""" + if not self._isOpen: raise portNotOpenError + # nothing done + return len(data) + + def flushInput(self): + """Clear input buffer, discarding all that is in the buffer.""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('ignored flushInput') + + def flushOutput(self): + """Clear output buffer, aborting the current output and + discarding all that is in the buffer.""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('ignored flushOutput') + + def sendBreak(self, duration=0.25): + """Send break condition. Timed, returns to idle state after given + duration.""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('ignored sendBreak({!r})'.format(duration)) + + def setBreak(self, level=True): + """Set break: Controls TXD. When active, to transmitting is + possible.""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('ignored setBreak({!r})'.format(level)) + + def setRTS(self, level=True): + """Set terminal status line: Request To Send""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('ignored setRTS({!r})'.format(level)) + + def setDTR(self, level=True): + """Set terminal status line: Data Terminal Ready""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('ignored setDTR({!r})'.format(level)) + + def getCTS(self): + """Read terminal status line: Clear To Send""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('returning dummy for getCTS()') + return True + + def getDSR(self): + """Read terminal status line: Data Set Ready""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('returning dummy for getDSR()') + return True + + def getRI(self): + """Read terminal status line: Ring Indicator""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('returning dummy for getRI()') + return False + + def getCD(self): + """Read terminal status line: Carrier Detect""" + if not self._isOpen: raise portNotOpenError + if self.logger: + self.logger.info('returning dummy for getCD()') + return True From c454c59c31fd1ebadf9afa22c18709c0c2d27c05 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Mon, 27 Nov 2017 17:17:43 -0500 Subject: [PATCH 02/12] Beginnings of a test of pocs/utils/rs232.py, utilizing PySerial's feature that allows a handler to be used based on the url of the serial "device". --- pocs/tests/serial_handlers/protocol_test.py | 62 ++++++++++++------- pocs/tests/test_rs232.py | 29 +++++++++ pocs/utils/rs232.py | 67 ++++++++++----------- 3 files changed, 103 insertions(+), 55 deletions(-) create mode 100644 pocs/tests/test_rs232.py diff --git a/pocs/tests/serial_handlers/protocol_test.py b/pocs/tests/serial_handlers/protocol_test.py index bd2125199..ccb676384 100644 --- a/pocs/tests/serial_handlers/protocol_test.py +++ b/pocs/tests/serial_handlers/protocol_test.py @@ -18,7 +18,8 @@ 'info': logging.INFO, 'warning': logging.WARNING, 'error': logging.ERROR, - } +} + class Serial(SerialBase, io.RawIOBase): """Serial port implementation for plain sockets.""" @@ -28,7 +29,8 @@ def open(self): if the port cannot be opened.""" self.logger = None if self._port is None: - raise SerialException("Port must be configured before it can be used.") + raise SerialException( + "Port must be configured before it can be used.") # not that there anything to configure... self._reconfigurePort() # all things set up get, now a clean start @@ -46,11 +48,13 @@ def close(self): self._isOpen = False def makeDeviceName(self, port): - raise SerialException("there is no sensible way to turn numbers into URLs") + raise SerialException( + "there is no sensible way to turn numbers into URLs") def fromURL(self, url): """extract host and port from an URL string""" - if url.lower().startswith("test://"): url = url[7:] + if url.lower().startswith("test://"): + url = url[7:] try: # is there a "path" (our options)? if '/' in url: @@ -63,105 +67,121 @@ def fromURL(self, url): else: value = None if option == 'logging': - logging.basicConfig() # XXX is that good to call it here? + logging.basicConfig( + ) # XXX is that good to call it here? self.logger = logging.getLogger('pySerial.test') self.logger.setLevel(LOGGER_LEVELS[value]) self.logger.debug('enabled logging') else: raise ValueError('unknown option: {!r}'.format(option)) except ValueError as e: - raise SerialException('expected a string in the form "[test://][option[/option...]]": {}'.format(e)) + raise SerialException( + 'expected a string in the form "[test://][option[/option...]]": {}'. + format(e)) return (host, port) # - - - - - - - - - - - - - - - - - - - - - - - - def inWaiting(self): """Return the number of characters currently in the input buffer.""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: # set this one to debug as the function could be called often... self.logger.debug('WARNING: inWaiting returns dummy value') - return 0 # hmmm, see comment in read() + return 0 # hmmm, see comment in read() def read(self, size=1): """Read size bytes from the serial port. If a timeout is set it may return less characters as requested. With no timeout it will block until the requested number of bytes is read.""" - if not self._isOpen: raise portNotOpenError - data = '123' # dummy data + if not self._isOpen: + raise portNotOpenError + data = '123' # dummy data return bytes(data) def write(self, data): """Output the given string over the serial port. Can block if the connection is blocked. May raise SerialException if the connection is closed.""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError # nothing done return len(data) def flushInput(self): """Clear input buffer, discarding all that is in the buffer.""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('ignored flushInput') def flushOutput(self): """Clear output buffer, aborting the current output and discarding all that is in the buffer.""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('ignored flushOutput') def sendBreak(self, duration=0.25): """Send break condition. Timed, returns to idle state after given duration.""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('ignored sendBreak({!r})'.format(duration)) def setBreak(self, level=True): """Set break: Controls TXD. When active, to transmitting is possible.""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('ignored setBreak({!r})'.format(level)) def setRTS(self, level=True): """Set terminal status line: Request To Send""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('ignored setRTS({!r})'.format(level)) def setDTR(self, level=True): """Set terminal status line: Data Terminal Ready""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('ignored setDTR({!r})'.format(level)) def getCTS(self): """Read terminal status line: Clear To Send""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('returning dummy for getCTS()') return True def getDSR(self): """Read terminal status line: Data Set Ready""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('returning dummy for getDSR()') return True def getRI(self): """Read terminal status line: Ring Indicator""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('returning dummy for getRI()') return False def getCD(self): """Read terminal status line: Carrier Detect""" - if not self._isOpen: raise portNotOpenError + if not self._isOpen: + raise portNotOpenError if self.logger: self.logger.info('returning dummy for getCD()') return True diff --git a/pocs/tests/test_rs232.py b/pocs/tests/test_rs232.py new file mode 100644 index 000000000..e805f8e0e --- /dev/null +++ b/pocs/tests/test_rs232.py @@ -0,0 +1,29 @@ +import pytest +import serial # PySerial, from https://github.com/pyserial/pyserial + +from pocs.utils import rs232 +from pocs.utils.config import load_config + +from pocs.tests.serial_handlers import protocol_test + + +@pytest.fixture(scope="module") +def handler(): + # Install our test handlers for the duration + serial.protocol_handler_packages.append('pocs.tests.serial_handlers') + yield True + # Remove our test handlers + serial.protocol_handler_packages.remove('pocs.tests.serial_handlers') + + +def test_basic(handler): + """ """ + ser = rs232.SerialData(port="test://") + assert ser.ser + assert not ser.is_connected + + +def test_bad_handler(handler): + """ """ + with pytest.raises(ValueError): + rs232.SerialData(port="another://") diff --git a/pocs/utils/rs232.py b/pocs/utils/rs232.py index 87b38034e..d1ecda52a 100644 --- a/pocs/utils/rs232.py +++ b/pocs/utils/rs232.py @@ -20,42 +20,37 @@ class SerialData(PanBase): def __init__(self, port=None, baudrate=115200, threaded=True, name="serial_data"): PanBase.__init__(self) - try: - self.ser = serial.Serial() - self.ser.port = port - self.ser.baudrate = baudrate - self.is_threaded = threaded - - self.ser.bytesize = serial.EIGHTBITS - self.ser.parity = serial.PARITY_NONE - self.ser.stopbits = serial.STOPBITS_ONE - self.ser.timeout = 1.0 - self.ser.xonxoff = False - self.ser.rtscts = False - self.ser.dsrdtr = False - self.ser.write_timeout = False - self.ser.open() - - self.name = name - self.queue = deque([], 1) - self._is_listening = False - self.loop_delay = 2. + self.ser = serial.serial_for_url(port, do_not_open=True) + self.ser.baudrate = baudrate + self.is_threaded = threaded + + self.ser.bytesize = serial.EIGHTBITS + self.ser.parity = serial.PARITY_NONE + self.ser.stopbits = serial.STOPBITS_ONE + self.ser.timeout = 1.0 + self.ser.xonxoff = False + self.ser.rtscts = False + self.ser.dsrdtr = False + self.ser.write_timeout = False + self.ser.open() + + self.name = name + self.queue = deque([], 1) + self._is_listening = False + self.loop_delay = 2. - if self.is_threaded: - self._serial_io = TextIOWrapper(BufferedRWPair(self.ser, self.ser), - newline='\r\n', encoding='ascii', line_buffering=True) + if self.is_threaded: + self._serial_io = TextIOWrapper(BufferedRWPair(self.ser, self.ser), + newline='\r\n', encoding='ascii', line_buffering=True) - self.logger.debug("Using threads (multiprocessing)") - self.process = Thread(target=self.receiving_function, args=(self.queue,)) - self.process.daemon = True - self.process.name = "PANOPTES_{}".format(name) + self.logger.debug("Using threads (multiprocessing)") + self.process = Thread(target=self.receiving_function, args=(self.queue,)) + self.process.daemon = True + self.process.name = "PANOPTES_{}".format(name) - self.logger.debug('Serial connection set up to {}, sleeping for two seconds'.format(self.name)) - time.sleep(2) - self.logger.debug('SerialData created') - except Exception as err: - self.ser = None - self.logger.critical('Could not set up serial port {} {}'.format(port, err)) + self.logger.debug('Serial connection set up to {}, sleeping for two seconds'.format(self.name)) + time.sleep(2) + self.logger.debug('SerialData created') @property def is_connected(self): @@ -197,5 +192,9 @@ def clear_buffer(self): # self.logger.debug('Cleared {} bytes from buffer'.format(count)) def __del__(self): - if self.ser: + try: + ser = self.ser + except NameError: + return + if ser: self.ser.close() From ba2a8c614599048462aabe1ecb0c64dbb8814121 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Mon, 27 Nov 2017 18:53:07 -0500 Subject: [PATCH 03/12] test_rs232.py works w.r.t. installing serial handler for test:// --- pocs/tests/test_rs232.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pocs/tests/test_rs232.py b/pocs/tests/test_rs232.py index e805f8e0e..68507313a 100644 --- a/pocs/tests/test_rs232.py +++ b/pocs/tests/test_rs232.py @@ -7,7 +7,7 @@ from pocs.tests.serial_handlers import protocol_test -@pytest.fixture(scope="module") +@pytest.fixture(scope="function") def handler(): # Install our test handlers for the duration serial.protocol_handler_packages.append('pocs.tests.serial_handlers') @@ -17,13 +17,20 @@ def handler(): def test_basic(handler): - """ """ + """Confirm we can create the SerialData object.""" ser = rs232.SerialData(port="test://") assert ser.ser assert not ser.is_connected + assert not ser.is_listening -def test_bad_handler(handler): - """ """ +def test_without_handler(): + """If the handlers aren't installed, it should fail.""" + with pytest.raises(ValueError): + rs232.SerialData(port="test://") + + +def test_another_handler(handler): + """And even when installed, the wrong name won't work.""" with pytest.raises(ValueError): rs232.SerialData(port="another://") From 38ec2cc29a027679a0a5037f03595cf9180ee390 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Tue, 28 Nov 2017 15:29:50 -0500 Subject: [PATCH 04/12] wip - Continuing effort to test uses of PySerial. --- pocs/tests/serial_handlers/__init__.py | 109 +++++++++ pocs/tests/serial_handlers/protocol_no_op.py | 123 ++++++++++ pocs/tests/serial_handlers/protocol_test.py | 244 ++++++++++++------- pocs/tests/test_rs232.py | 5 + 4 files changed, 387 insertions(+), 94 deletions(-) create mode 100644 pocs/tests/serial_handlers/protocol_no_op.py diff --git a/pocs/tests/serial_handlers/__init__.py b/pocs/tests/serial_handlers/__init__.py index e69de29bb..e43ef96c1 100644 --- a/pocs/tests/serial_handlers/__init__.py +++ b/pocs/tests/serial_handlers/__init__.py @@ -0,0 +1,109 @@ +# The protocol_*.py files in this package are based on PySerial's file +# test/handlers/protocol_test.py, modified for different behaviors. +# The call serial.serial_for_url("XYZ://") looks for a class Serial in a +# file named protocol_XYZ.py in this package (i.e. directory). + +from serial.serialutil import SerialBase, SerialException, portNotOpenError + +# A base class which handles lots of the common behaviors, especially properties +# that we probably don't care about for our tests. + +class NoOpSerialBase(SerialBase): + """No-op implementation of PySerial's SerialBase. + + Provides no-op implementation of various methods that SerialBase expects + to have implemented by the sub-class. Can be used as is for a /dev/null + type of behavior. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def open(self): + """Open port. + + Raises: + SerialException if the port cannot be opened. + """ + self.is_open = True + + def close(self): + """Close port immediately.""" + self.is_open = False + + def read(self, size=1): + """Read size bytes. + + If a timeout is set it may return fewer characters than requested. + With no timeout it will block until the requested number of bytes + is read. + + Args: + size: Number of bytes to read. + + Returns: + Bytes read from the port, of type 'bytes'. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + if not self.is_open: + raise portNotOpenError + return bytes() + + @abstractmethod + def write(self, data): + """ + Args: + data: The data to write. + + Returns: + Number of bytes written. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + if not self.is_open: + raise portNotOpenError + return len(data) + + #--------------------------------------------------------------------------- + # There are a number of methods called by SerialBase that need to be + # implemented by sub-classes, assuming their calls haven't been blocked + # by replacing the calling methods/properties. These are no-op + # implementations. + + def _reconfigure_port(self): + """Reconfigure the open port after a property has been changed. + + If you need to know which property has been changed, override the + setter for the appropriate properties. + """ + pass + + def _update_rts_state(self): + """Handle rts being set to some value. + + "self.rts = value" has been executed, for some value. This may not + have changed the value. + """ + pass + + def _update_dtr_state(self): + """Handle dtr being set to some value. + + "self.dtr = value" has been executed, for some value. This may not + have changed the value. + """ + pass + + def _update_break_state(self): + """Handle break_condition being set to some value. + + "self.break_condition = value" has been executed, for some value. + This may not have changed the value. + Note that break_condition is set and then cleared by send_break(). + """ + pass diff --git a/pocs/tests/serial_handlers/protocol_no_op.py b/pocs/tests/serial_handlers/protocol_no_op.py new file mode 100644 index 000000000..1c0f156f6 --- /dev/null +++ b/pocs/tests/serial_handlers/protocol_no_op.py @@ -0,0 +1,123 @@ +# This module implements a handler for serial_for_url("no_op://"). + +import io +from serial.serialutil import SerialBase, SerialException, portNotOpenError + + +class Serial(SerialBase, io.RawIOBase): + """Pseudo-Serial class for url test://""" + + def __init__(self, *args, **kwargs): + self.logger = None + super().__init__(*args, **kwargs) + + def open(self): + """Open port. + + Raises: + SerialException if the port cannot be opened. + """ + self._isOpen = True + + def close(self): + """Close port""" + self._isOpen = False + + def read(self, size=1): + """Read size bytes from the serial port.""" + if not self._isOpen: + raise portNotOpenError + return bytes() + + def write(self, data): + """Write the given data to the serial port.""" + if not self._isOpen: + raise portNotOpenError + return len(data) + + # - - - - - - - - - - - - - - - - - - - - - - - - + + @property + def in_waiting(self): + """Return the number of characters currently in the input buffer.""" + if not self._isOpen: + raise portNotOpenError + return 0 # Nothing waiting. + + def reset_input_buffer(self): + """Flush input buffer, discarding all it’s contents.""" + if not self._isOpen: + raise portNotOpenError + + def reset_output_buffer(self): + """Clear output buffer. + + Clear output buffer, aborting the current output and discarding + all that is in the buffer. + """ + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('ignored flushOutput') + + def send_break(self, duration=0.25): + """Send break condition. Timed, returns to idle state after given + duration.""" + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('ignored sendBreak({!r})'.format(duration)) + + def setBreak(self, level=True): + """Set break: Controls TXD. When active, to transmitting is + possible.""" + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('ignored setBreak({!r})'.format(level)) + + def setRTS(self, level=True): + """Set terminal status line: Request To Send""" + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('ignored setRTS({!r})'.format(level)) + + def setDTR(self, level=True): + """Set terminal status line: Data Terminal Ready""" + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('ignored setDTR({!r})'.format(level)) + + def getCTS(self): + """Read terminal status line: Clear To Send""" + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('returning dummy for getCTS()') + return True + + def getDSR(self): + """Read terminal status line: Data Set Ready""" + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('returning dummy for getDSR()') + return True + + def getRI(self): + """Read terminal status line: Ring Indicator""" + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('returning dummy for getRI()') + return False + + def getCD(self): + """Read terminal status line: Carrier Detect""" + if not self._isOpen: + raise portNotOpenError + if self.logger: + self.logger.info('returning dummy for getCD()') + return True diff --git a/pocs/tests/serial_handlers/protocol_test.py b/pocs/tests/serial_handlers/protocol_test.py index ccb676384..0884c8565 100644 --- a/pocs/tests/serial_handlers/protocol_test.py +++ b/pocs/tests/serial_handlers/protocol_test.py @@ -5,126 +5,177 @@ # Apparently the file name needs to be protocol_, # where here that is "test", allowing URL's of the format "test://". -from serial.serialutil import SerialBase, SerialException, portNotOpenError - +from abc import ABCMeta, abstractmethod, abstractproperty import io -import logging -import socket +from serial.serialutil import SerialBase, SerialException, portNotOpenError import time -# map log level names to constants. used in fromURL() -LOGGER_LEVELS = { - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warning': logging.WARNING, - 'error': logging.ERROR, -} +the_hook_creator = None -class Serial(SerialBase, io.RawIOBase): - """Serial port implementation for plain sockets.""" +class AbstractSerialHooks(object): + def __init__(self, ser, *args, **kwargs): + self.ser = ser + + def created(self): + pass def open(self): - """Open port with current settings. This may throw a SerialException - if the port cannot be opened.""" - self.logger = None - if self._port is None: - raise SerialException( - "Port must be configured before it can be used.") - # not that there anything to configure... - self._reconfigurePort() - # all things set up get, now a clean start - self._isOpen = True - - def _reconfigurePort(self): - """Set communication parameters on opened port. for the test:// - protocol all settings are ignored!""" - if self.logger: - self.logger.info('ignored port configuration change') + """Open port. + + Raises: + SerialException if the port cannot be opened. + """ + self.ser._isOpen = True def close(self): """Close port""" - if self._isOpen: - self._isOpen = False - - def makeDeviceName(self, port): - raise SerialException( - "there is no sensible way to turn numbers into URLs") - - def fromURL(self, url): - """extract host and port from an URL string""" - if url.lower().startswith("test://"): - url = url[7:] - try: - # is there a "path" (our options)? - if '/' in url: - # cut away options - url, options = url.split('/', 1) - # process options now, directly altering self - for option in options.split('/'): - if '=' in option: - option, value = option.split('=', 1) - else: - value = None - if option == 'logging': - logging.basicConfig( - ) # XXX is that good to call it here? - self.logger = logging.getLogger('pySerial.test') - self.logger.setLevel(LOGGER_LEVELS[value]) - self.logger.debug('enabled logging') - else: - raise ValueError('unknown option: {!r}'.format(option)) - except ValueError as e: - raise SerialException( - 'expected a string in the form "[test://][option[/option...]]": {}'. - format(e)) - return (host, port) + self.ser._isOpen = False - # - - - - - - - - - - - - - - - - - - - - - - - - + @abstractmethod + def read(self, size=1): + """Read size bytes. + + If a timeout is set it may return fewer characters than requested. + With no timeout it will block until the requested number of bytes + is read. - def inWaiting(self): - """Return the number of characters currently in the input buffer.""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - # set this one to debug as the function could be called often... - self.logger.debug('WARNING: inWaiting returns dummy value') - return 0 # hmmm, see comment in read() + Args: + size: Number of bytes to read. + + Returns: + Bytes read from the port, of type 'bytes'. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + return NotImplemented + + @abstractmethod + def write(self, data): + """ + Args: + data: The data to write. + + Returns: + Number of bytes written. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + return NotImplemented + + +class SimpleSerialHooks(object): + def __init__(self, ser, read_from_bytes, *args, **kwargs): + super().__init__(ser, *args, **kwargs) + self.read_from_bytes = read_from_bytes + self.write_to_byte_array = bytearray() def read(self, size=1): - """Read size bytes from the serial port. If a timeout is set it may - return less characters as requested. With no timeout it will block - until the requested number of bytes is read.""" + if size >= len(self.read_from_bytes): + result = bytes(self.read_from_bytes) + self.read_from_bytes = "" + return result + else: + result = bytes(self.read_from_bytes[0:size]) + self.read_from_bytes = del self.read_from_bytes[0:size] + return result + + def write(self, data): + if not isinstance(data, (bytes, bytearray)): + data = bytes(data) + self.write_to_byte_array.extend(data) + return len(data) + + +def RegisterHookCreator(fn): + global the_hook_creator + the_hook_creator = fn + + +def ClearHookCreator(): + global the_hook_creator + the_hook_creator = None + + +class Serial(SerialBase, io.RawIOBase): + """Pseudo-Serial class for url test://""" + + def __init__(self, *args, **kwargs): + # Read-write property values + self._break_condition = False + self._rts = False + self._dtr = False + # Read-only property values + self._name = False + self._rts = False + self._ctx = False + self._dsr = False + self._ri = False + self._ri = False + self._ri = False + self._ri = False + self.logger = None + self.hooks = the_hook_creator(self, *args, **kwargs) + super().__init__(*args, **kwargs) + self.hooks.created(self) + + def open(self): + """Open port. + + Raises: + SerialException if the port cannot be opened. + """ + self.hooks.open() + + def close(self): + """Close port""" + self.hooks.close() + + def read(self, size=1): + """Read size bytes from the serial port.""" if not self._isOpen: raise portNotOpenError - data = '123' # dummy data - return bytes(data) + return self.hooks.read(size) def write(self, data): - """Output the given string over the serial port. Can block if the - connection is blocked. May raise SerialException if the connection is - closed.""" + """Write the given data to the serial port.""" if not self._isOpen: raise portNotOpenError - # nothing done - return len(data) + return self.hooks.write(data) + + # - - - - - - - - - - - - - - - - - - - - - - - - - def flushInput(self): - """Clear input buffer, discarding all that is in the buffer.""" + @property + def in_waiting(self): + """Return the number of characters currently in the input buffer.""" if not self._isOpen: raise portNotOpenError if self.logger: - self.logger.info('ignored flushInput') + # set this one to debug as the function could be called often... + self.logger.debug('WARNING: in_waiting returns dummy value') + return 0 # hmmm, see comment in read() - def flushOutput(self): - """Clear output buffer, aborting the current output and - discarding all that is in the buffer.""" + def reset_input_buffer(self): + """Flush input buffer, discarding all it’s contents.""" + if not self._isOpen: + raise portNotOpenError + + def reset_output_buffer(self): + """Clear output buffer. + + Clear output buffer, aborting the current output and discarding + all that is in the buffer. + """ if not self._isOpen: raise portNotOpenError if self.logger: self.logger.info('ignored flushOutput') - def sendBreak(self, duration=0.25): + def send_break(self, duration=0.25): """Send break condition. Timed, returns to idle state after given duration.""" if not self._isOpen: @@ -132,11 +183,16 @@ def sendBreak(self, duration=0.25): if self.logger: self.logger.info('ignored sendBreak({!r})'.format(duration)) - def setBreak(self, level=True): - """Set break: Controls TXD. When active, to transmitting is + @property + def break_condition(self): + """Get break_condition.""" + return self.__break_condition + + @break_condition.setter + def break_condition(self, value): + """Set break_condition: Controls TXD. When active, to transmitting is possible.""" - if not self._isOpen: - raise portNotOpenError + self.__break_condition = value if self.logger: self.logger.info('ignored setBreak({!r})'.format(level)) diff --git a/pocs/tests/test_rs232.py b/pocs/tests/test_rs232.py index 68507313a..46dc5beb9 100644 --- a/pocs/tests/test_rs232.py +++ b/pocs/tests/test_rs232.py @@ -6,6 +6,11 @@ from pocs.tests.serial_handlers import protocol_test +THE_HOOKS = None +def create_hooks(ser, *args, **kwargs): + THE_HOOKS = SimpleSerialHooks(ser, + + @pytest.fixture(scope="function") def handler(): From bb1279dcd6b1a5d638048c1f6ea540f4e463bdae Mon Sep 17 00:00:00 2001 From: jamessynge Date: Tue, 28 Nov 2017 21:57:14 -0500 Subject: [PATCH 05/12] wip - Added protocol_buffers.py, but it isn't quite working. I think that the module is being loaded twice, once by test_rs232.py, and another time by serial.serial_for_url(). As a result, the buffers set by the test aren't visible to the BuffersSerial instance. --- pocs/tests/serial_handlers/__init__.py | 18 +-- .../tests/serial_handlers/protocol_buffers.py | 92 +++++++++++++ pocs/tests/serial_handlers/protocol_no_op.py | 123 +----------------- pocs/tests/serial_handlers/protocol_test.py | 2 +- pocs/tests/test_rs232.py | 54 +++++++- pocs/utils/rs232.py | 75 ++++++----- 6 files changed, 199 insertions(+), 165 deletions(-) create mode 100644 pocs/tests/serial_handlers/protocol_buffers.py diff --git a/pocs/tests/serial_handlers/__init__.py b/pocs/tests/serial_handlers/__init__.py index e43ef96c1..246cad8c9 100644 --- a/pocs/tests/serial_handlers/__init__.py +++ b/pocs/tests/serial_handlers/__init__.py @@ -3,12 +3,10 @@ # The call serial.serial_for_url("XYZ://") looks for a class Serial in a # file named protocol_XYZ.py in this package (i.e. directory). -from serial.serialutil import SerialBase, SerialException, portNotOpenError +from serial import serialutil -# A base class which handles lots of the common behaviors, especially properties -# that we probably don't care about for our tests. -class NoOpSerialBase(SerialBase): +class NoOpSerial(serialutil.SerialBase): """No-op implementation of PySerial's SerialBase. Provides no-op implementation of various methods that SerialBase expects @@ -19,6 +17,11 @@ class NoOpSerialBase(SerialBase): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + @property + def in_waiting(self): + """The number of input bytes available to read immediately.""" + return 0 + def open(self): """Open port. @@ -49,10 +52,9 @@ def read(self, size=1): the port and the time is exceeded. """ if not self.is_open: - raise portNotOpenError + raise serialutil.portNotOpenError return bytes() - @abstractmethod def write(self, data): """ Args: @@ -66,8 +68,8 @@ def write(self, data): the port and the time is exceeded. """ if not self.is_open: - raise portNotOpenError - return len(data) + raise serialutil.portNotOpenError + return 0 #--------------------------------------------------------------------------- # There are a number of methods called by SerialBase that need to be diff --git a/pocs/tests/serial_handlers/protocol_buffers.py b/pocs/tests/serial_handlers/protocol_buffers.py new file mode 100644 index 000000000..99a425ddc --- /dev/null +++ b/pocs/tests/serial_handlers/protocol_buffers.py @@ -0,0 +1,92 @@ +# This module implements a handler for serial_for_url("buffers://"). + +from pocs.tests.serial_handlers import NoOpSerial + +import io +import threading + +# r_buffer and w_buffer are binary I/O buffers. read(size=N) on an instance +# of Serial reads the next N bytes from r_buffer, and write(data) appends the +# bytes of data to w_buffer. +# NOTE: The caller (a test) is responsible for resetting buffers before tests. +_r_buffer = None # io.BytesIO() +_w_buffer = None # io.BytesIO() + +# The above I/O buffers are not thread safe, so we need to lock them during +# access. +_r_lock = threading.Lock() +_w_lock = threading.Lock() + + +def ResetBuffers(read_data=None): + SetRBufferValue(read_data) + with _w_lock: + _w_buffer = io.BytesIO() + + +def SetRBufferValue(data): + """Sets the r buffer to data (a bytes object).""" + if data and not isinstance(data, (bytes, bytearray)): + raise TypeError("data must by a bytes or bytearray object.") + with _r_lock: + _r_buffer = io.BytesIO(data) + + +def GetWBufferValue(): + """Returns an immutable bytes object with the value of the w buffer.""" + with _w_lock: + if _w_buffer: + return _w_buffer.getvalue() + + +class BuffersSerial(NoOpSerial): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def read(self, size=1): + """Read size bytes. + + If a timeout is set it may return fewer characters than requested. + With no timeout it will block until the requested number of bytes + is read. + + Args: + size: Number of bytes to read. + + Returns: + Bytes read from the port, of type 'bytes'. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + if not self.is_open: + raise serialutil.portNotOpenError + with _r_lock: + # TODO(jamessynge): Figure out whether and how to handle timeout. + # We might choose to generate a timeout if the caller asks for data + # beyond the end of the buffer; or simply return what is left, + # including nothing (i.e. bytes()) if there is nothing left. + return _r_buffer.read(size) + + def write(self, data): + """ + Args: + data: The data to write. + + Returns: + Number of bytes written. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + if not isinstance(data, (bytes, bytearray)): + raise TypeError("data must by a bytes or bytearray object.") + if not self.is_open: + raise serialutil.portNotOpenError + with _w_lock: + return _w_buffer.write(data) + + +Serial = BuffersSerial diff --git a/pocs/tests/serial_handlers/protocol_no_op.py b/pocs/tests/serial_handlers/protocol_no_op.py index 1c0f156f6..4af6c9396 100644 --- a/pocs/tests/serial_handlers/protocol_no_op.py +++ b/pocs/tests/serial_handlers/protocol_no_op.py @@ -1,123 +1,6 @@ # This module implements a handler for serial_for_url("no_op://"). -import io -from serial.serialutil import SerialBase, SerialException, portNotOpenError +from pocs.tests.serial_handlers import NoOpSerial - -class Serial(SerialBase, io.RawIOBase): - """Pseudo-Serial class for url test://""" - - def __init__(self, *args, **kwargs): - self.logger = None - super().__init__(*args, **kwargs) - - def open(self): - """Open port. - - Raises: - SerialException if the port cannot be opened. - """ - self._isOpen = True - - def close(self): - """Close port""" - self._isOpen = False - - def read(self, size=1): - """Read size bytes from the serial port.""" - if not self._isOpen: - raise portNotOpenError - return bytes() - - def write(self, data): - """Write the given data to the serial port.""" - if not self._isOpen: - raise portNotOpenError - return len(data) - - # - - - - - - - - - - - - - - - - - - - - - - - - - - @property - def in_waiting(self): - """Return the number of characters currently in the input buffer.""" - if not self._isOpen: - raise portNotOpenError - return 0 # Nothing waiting. - - def reset_input_buffer(self): - """Flush input buffer, discarding all it’s contents.""" - if not self._isOpen: - raise portNotOpenError - - def reset_output_buffer(self): - """Clear output buffer. - - Clear output buffer, aborting the current output and discarding - all that is in the buffer. - """ - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored flushOutput') - - def send_break(self, duration=0.25): - """Send break condition. Timed, returns to idle state after given - duration.""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored sendBreak({!r})'.format(duration)) - - def setBreak(self, level=True): - """Set break: Controls TXD. When active, to transmitting is - possible.""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored setBreak({!r})'.format(level)) - - def setRTS(self, level=True): - """Set terminal status line: Request To Send""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored setRTS({!r})'.format(level)) - - def setDTR(self, level=True): - """Set terminal status line: Data Terminal Ready""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored setDTR({!r})'.format(level)) - - def getCTS(self): - """Read terminal status line: Clear To Send""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('returning dummy for getCTS()') - return True - - def getDSR(self): - """Read terminal status line: Data Set Ready""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('returning dummy for getDSR()') - return True - - def getRI(self): - """Read terminal status line: Ring Indicator""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('returning dummy for getRI()') - return False - - def getCD(self): - """Read terminal status line: Carrier Detect""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('returning dummy for getCD()') - return True +# Export it as Serial so that it will be picked up by PySerial's serial_for_url. +Serial = NoOpSerial diff --git a/pocs/tests/serial_handlers/protocol_test.py b/pocs/tests/serial_handlers/protocol_test.py index 0884c8565..29f04f8c5 100644 --- a/pocs/tests/serial_handlers/protocol_test.py +++ b/pocs/tests/serial_handlers/protocol_test.py @@ -81,7 +81,7 @@ def read(self, size=1): return result else: result = bytes(self.read_from_bytes[0:size]) - self.read_from_bytes = del self.read_from_bytes[0:size] + del self.read_from_bytes[0:size] return result def write(self, data): diff --git a/pocs/tests/test_rs232.py b/pocs/tests/test_rs232.py index 46dc5beb9..281956735 100644 --- a/pocs/tests/test_rs232.py +++ b/pocs/tests/test_rs232.py @@ -4,15 +4,18 @@ from pocs.utils import rs232 from pocs.utils.config import load_config +from pocs.tests.serial_handlers import protocol_buffers +from pocs.tests.serial_handlers import protocol_no_op from pocs.tests.serial_handlers import protocol_test THE_HOOKS = None -def create_hooks(ser, *args, **kwargs): - THE_HOOKS = SimpleSerialHooks(ser, +def create_hooks(ser, *args, **kwargs): + THE_HOOKS = SimpleSerialHooks(ser) + -@pytest.fixture(scope="function") +@pytest.fixture(scope='function') def handler(): # Install our test handlers for the duration serial.protocol_handler_packages.append('pocs.tests.serial_handlers') @@ -22,9 +25,22 @@ def handler(): def test_basic(handler): - """Confirm we can create the SerialData object.""" - ser = rs232.SerialData(port="test://") + # Confirm we can create the SerialData object. + ser = rs232.SerialData(port='no_op://', open_delay=0) + # Peek inside, it should have a PySerial instance as member ser. assert ser.ser + assert ser.ser.__class__.__name__ == 'NoOpSerial' + print(str(ser.ser)) + # Open is automatically called by SerialData. + assert ser.is_connected + # Not using threading. + assert not ser.is_listening + + # no_op handler doesn't do any reading or writing. + assert '' == ser.read(retry_delay=0.01, retry_limit=2) + assert 0 == ser.write('') + + ser.disconnect() assert not ser.is_connected assert not ser.is_listening @@ -32,10 +48,34 @@ def test_basic(handler): def test_without_handler(): """If the handlers aren't installed, it should fail.""" with pytest.raises(ValueError): - rs232.SerialData(port="test://") + rs232.SerialData(port='no_op://') def test_another_handler(handler): """And even when installed, the wrong name won't work.""" with pytest.raises(ValueError): - rs232.SerialData(port="another://") + rs232.SerialData(port='bogus://') + + +def test_unthreaded_io(handler): + pytest.set_trace() + protocol_buffers.ResetBuffers(b'abc\r\n') + ser = rs232.SerialData(port='buffers://', open_delay=0) + assert ser + # Peek inside, it should have a PySerial instance as member ser. + assert ser.ser + assert ser.ser.__class__.__name__ == 'BuffersSerial' + print(str(ser.ser)) + # Open is automatically called by SerialData. + assert ser.is_connected + # Not using threading. + assert not ser.is_listening + + # no_op handler doesn't do any reading or writing. + assert 'abc' == ser.read(retry_delay=0.01, retry_limit=2) + assert 5 == ser.write(b'def\r\n') + + ser.disconnect() + assert not ser.is_connected + assert not ser.is_listening + diff --git a/pocs/utils/rs232.py b/pocs/utils/rs232.py index d1ecda52a..38262a8ba 100644 --- a/pocs/utils/rs232.py +++ b/pocs/utils/rs232.py @@ -12,12 +12,16 @@ class SerialData(PanBase): - """ Main serial class """ - def __init__(self, port=None, baudrate=115200, threaded=True, name="serial_data"): + def __init__(self, + port=None, + baudrate=115200, + threaded=True, + name="serial_data", + open_delay=2.0): PanBase.__init__(self) self.ser = serial.serial_for_url(port, do_not_open=True) @@ -32,31 +36,42 @@ def __init__(self, port=None, baudrate=115200, threaded=True, name="serial_data" self.ser.rtscts = False self.ser.dsrdtr = False self.ser.write_timeout = False - self.ser.open() self.name = name self.queue = deque([], 1) self._is_listening = False + self.process = None + self._serial_io = None self.loop_delay = 2. + # Properties have been set to reasonable values, ready to open the port. + self.ser.open() + if self.is_threaded: - self._serial_io = TextIOWrapper(BufferedRWPair(self.ser, self.ser), - newline='\r\n', encoding='ascii', line_buffering=True) + self._serial_io = TextIOWrapper( + BufferedRWPair(self.ser, self.ser), + newline='\r\n', + encoding='ascii', + line_buffering=True) self.logger.debug("Using threads (multiprocessing)") - self.process = Thread(target=self.receiving_function, args=(self.queue,)) + self.process = Thread( + target=self.receiving_function, args=(self.queue, )) self.process.daemon = True self.process.name = "PANOPTES_{}".format(name) - self.logger.debug('Serial connection set up to {}, sleeping for two seconds'.format(self.name)) - time.sleep(2) + # TODO(jamessynge): Consider eliminating this sleep period, or making + # it a configurable option. For one thing, it slows down tests! + self.logger.debug( + 'Serial connection set up to %r, sleeping for %r seconds', + self.name, open_delay) + if open_delay: + time.sleep(open_delay) self.logger.debug('SerialData created') @property def is_connected(self): - """ - Checks the serial connection on the mount to determine if connection is open - """ + """True if serial port is open, False otherwise.""" connected = False if self.ser: connected = self.ser.isOpen() @@ -69,13 +84,15 @@ def is_listening(self): def start(self): """ Starts the separate process """ - self.logger.debug("Starting serial process: {}".format(self.process.name)) + self.logger.debug("Starting serial process: {}".format( + self.process.name)) self._is_listening = True self.process.start() def stop(self): """ Starts the separate process """ - self.logger.debug("Stopping serial process: {}".format(self.process.name)) + self.logger.debug("Stopping serial process: {}".format( + self.process.name)) self._is_listening = False self.process.join() @@ -92,7 +109,8 @@ def connect(self): if not self.ser.isOpen(): raise BadSerialConnection(msg="Serial connection is not open") - self.logger.debug('Serial connection established to {}'.format(self.name)) + self.logger.debug('Serial connection established to {}'.format( + self.name)) return self.ser.isOpen() def disconnect(self): @@ -112,7 +130,8 @@ def receiving_function(self, q): ts = time.strftime('%Y-%m-%dT%H:%M:%S %Z', time.gmtime()) self.queue.append((ts, line)) except IOError as err: - self.logger.warning("Device is not sending messages. IOError: {}".format(err)) + self.logger.warning( + "Device is not sending messages. IOError: {}".format(err)) time.sleep(2) except UnicodeDecodeError: self.logger.warning("Unicode problem") @@ -137,27 +156,26 @@ def write(self, value): return response - def read(self): - """ - Reads value using readline - If no response is given, delay and then try to read again. Fail after 10 attempts + def read(self, retry_limit=5, retry_delay=0.5): + """Reads next line of input using readline. + + If no response is given, delay for retry_delay and then try to read + again. Fail after retry_limit attempts. """ assert self.ser assert self.ser.isOpen() - retry_limit = 5 - delay = 0.5 - while True and retry_limit: if self.is_threaded: response_string = self._serial_io.readline() else: - response_string = self.ser.readline(self.ser.inWaiting()).decode() + response_string = self.ser.readline( + self.ser.inWaiting()).decode() if response_string > '': break - time.sleep(delay) + time.sleep(retry_delay) retry_limit -= 1 # self.logger.debug('Serial read: {}'.format(response_string)) @@ -184,11 +202,10 @@ def get_reading(self): def clear_buffer(self): """ Clear Response Buffer """ - count = 0 - while self.ser.inWaiting() > 0: - count += 1 - self.ser.read(1) - + # Not worrying here about bytes arriving between reading in_waiting + # and calling reset_input_buffer(). + count = self.ser.in_waiting + self.ser.reset_input_buffer() # self.logger.debug('Cleared {} bytes from buffer'.format(count)) def __del__(self): From 6e4795de021e25cd0215c1188ab2cb0abbea3ca3 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Wed, 29 Nov 2017 21:34:13 -0500 Subject: [PATCH 06/12] wip - Almost working now, except for the threaded case which is hanging. --- pocs/tests/serial_handlers/__init__.py | 4 + .../tests/serial_handlers/protocol_buffers.py | 19 +- pocs/tests/serial_handlers/protocol_hooked.py | 30 +++ pocs/tests/serial_handlers/protocol_test.py | 243 ------------------ pocs/tests/test_rs232.py | 158 +++++++++--- pocs/utils/rs232.py | 34 ++- 6 files changed, 201 insertions(+), 287 deletions(-) create mode 100644 pocs/tests/serial_handlers/protocol_hooked.py delete mode 100644 pocs/tests/serial_handlers/protocol_test.py diff --git a/pocs/tests/serial_handlers/__init__.py b/pocs/tests/serial_handlers/__init__.py index 246cad8c9..0d221e7ea 100644 --- a/pocs/tests/serial_handlers/__init__.py +++ b/pocs/tests/serial_handlers/__init__.py @@ -3,6 +3,10 @@ # The call serial.serial_for_url("XYZ://") looks for a class Serial in a # file named protocol_XYZ.py in this package (i.e. directory). +print("Importing serial_handlers __init__.py") +print("__name__:", __name__) +print("__file__:", __file__) + from serial import serialutil diff --git a/pocs/tests/serial_handlers/protocol_buffers.py b/pocs/tests/serial_handlers/protocol_buffers.py index 99a425ddc..d9211df57 100644 --- a/pocs/tests/serial_handlers/protocol_buffers.py +++ b/pocs/tests/serial_handlers/protocol_buffers.py @@ -1,5 +1,9 @@ # This module implements a handler for serial_for_url("buffers://"). +print("Importing protocol_buffers.py") +print("__name__:", __name__) +print("__file__:", __file__) + from pocs.tests.serial_handlers import NoOpSerial import io @@ -9,8 +13,8 @@ # of Serial reads the next N bytes from r_buffer, and write(data) appends the # bytes of data to w_buffer. # NOTE: The caller (a test) is responsible for resetting buffers before tests. -_r_buffer = None # io.BytesIO() -_w_buffer = None # io.BytesIO() +_r_buffer = None +_w_buffer = None # The above I/O buffers are not thread safe, so we need to lock them during # access. @@ -21,6 +25,7 @@ def ResetBuffers(read_data=None): SetRBufferValue(read_data) with _w_lock: + global _w_buffer _w_buffer = io.BytesIO() @@ -29,6 +34,7 @@ def SetRBufferValue(data): if data and not isinstance(data, (bytes, bytearray)): raise TypeError("data must by a bytes or bytearray object.") with _r_lock: + global _r_buffer _r_buffer = io.BytesIO(data) @@ -43,9 +49,16 @@ class BuffersSerial(NoOpSerial): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + @property + def in_waiting(self): + if not self.is_open: + raise serialutil.portNotOpenError + with _r_lock: + return len(_r_buffer.getbuffer()) - _r_buffer.tell() + def read(self, size=1): """Read size bytes. - + If a timeout is set it may return fewer characters than requested. With no timeout it will block until the requested number of bytes is read. diff --git a/pocs/tests/serial_handlers/protocol_hooked.py b/pocs/tests/serial_handlers/protocol_hooked.py new file mode 100644 index 000000000..ba2a61f85 --- /dev/null +++ b/pocs/tests/serial_handlers/protocol_hooked.py @@ -0,0 +1,30 @@ +# This module enables a test to provide a handler for "hooked://..." urls +# passed into serial.serial_for_url. To do so, set the value of +# serial_class_for_url from your test to a function with the same API as +# ExampleSerialClassForUrl. + +from pocs.tests.serial_handlers import NoOpSerial + + +def ExampleSerialClassForUrl(url): + """Implementation of serial_class_for_url called by serial.serial_for_url. + + Returns the url, possibly modified, and a factory function to be called to + create an instance of a SerialBase sub-class (or at least behaves like it). + You can return a class as that factory function, as calling a class creates + an instance of that class. + + serial.serial_for_url will call that factory function with None as the + port parameter (the first), and after creating the instance will assign + the url to the port property of the instance. + + Returns: + A tuple (url, factory). + """ + return url, Serial + +# Assign to this global variable from a test to override this default behavior. +serial_class_for_url = ExampleSerialClassForUrl + +# Or assign your own class to this global variable. +Serial = NoOpSerial diff --git a/pocs/tests/serial_handlers/protocol_test.py b/pocs/tests/serial_handlers/protocol_test.py deleted file mode 100644 index 29f04f8c5..000000000 --- a/pocs/tests/serial_handlers/protocol_test.py +++ /dev/null @@ -1,243 +0,0 @@ -# This is based on PySerial's file test/handlers/protocol_test.py (i.e. is -# a copy of it, that I'll then modify for testing). -# -# This module implements a URL dummy handler for serial_for_url. -# Apparently the file name needs to be protocol_, -# where here that is "test", allowing URL's of the format "test://". - -from abc import ABCMeta, abstractmethod, abstractproperty -import io -from serial.serialutil import SerialBase, SerialException, portNotOpenError -import time - -the_hook_creator = None - - -class AbstractSerialHooks(object): - def __init__(self, ser, *args, **kwargs): - self.ser = ser - - def created(self): - pass - - def open(self): - """Open port. - - Raises: - SerialException if the port cannot be opened. - """ - self.ser._isOpen = True - - def close(self): - """Close port""" - self.ser._isOpen = False - - @abstractmethod - def read(self, size=1): - """Read size bytes. - - If a timeout is set it may return fewer characters than requested. - With no timeout it will block until the requested number of bytes - is read. - - Args: - size: Number of bytes to read. - - Returns: - Bytes read from the port, of type 'bytes'. - - Raises: - SerialTimeoutException: In case a write timeout is configured for - the port and the time is exceeded. - """ - return NotImplemented - - @abstractmethod - def write(self, data): - """ - Args: - data: The data to write. - - Returns: - Number of bytes written. - - Raises: - SerialTimeoutException: In case a write timeout is configured for - the port and the time is exceeded. - """ - return NotImplemented - - -class SimpleSerialHooks(object): - def __init__(self, ser, read_from_bytes, *args, **kwargs): - super().__init__(ser, *args, **kwargs) - self.read_from_bytes = read_from_bytes - self.write_to_byte_array = bytearray() - - def read(self, size=1): - if size >= len(self.read_from_bytes): - result = bytes(self.read_from_bytes) - self.read_from_bytes = "" - return result - else: - result = bytes(self.read_from_bytes[0:size]) - del self.read_from_bytes[0:size] - return result - - def write(self, data): - if not isinstance(data, (bytes, bytearray)): - data = bytes(data) - self.write_to_byte_array.extend(data) - return len(data) - - -def RegisterHookCreator(fn): - global the_hook_creator - the_hook_creator = fn - - -def ClearHookCreator(): - global the_hook_creator - the_hook_creator = None - - -class Serial(SerialBase, io.RawIOBase): - """Pseudo-Serial class for url test://""" - - def __init__(self, *args, **kwargs): - # Read-write property values - self._break_condition = False - self._rts = False - self._dtr = False - # Read-only property values - self._name = False - self._rts = False - self._ctx = False - self._dsr = False - self._ri = False - self._ri = False - self._ri = False - self._ri = False - self.logger = None - self.hooks = the_hook_creator(self, *args, **kwargs) - super().__init__(*args, **kwargs) - self.hooks.created(self) - - def open(self): - """Open port. - - Raises: - SerialException if the port cannot be opened. - """ - self.hooks.open() - - def close(self): - """Close port""" - self.hooks.close() - - def read(self, size=1): - """Read size bytes from the serial port.""" - if not self._isOpen: - raise portNotOpenError - return self.hooks.read(size) - - def write(self, data): - """Write the given data to the serial port.""" - if not self._isOpen: - raise portNotOpenError - return self.hooks.write(data) - - # - - - - - - - - - - - - - - - - - - - - - - - - - - @property - def in_waiting(self): - """Return the number of characters currently in the input buffer.""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - # set this one to debug as the function could be called often... - self.logger.debug('WARNING: in_waiting returns dummy value') - return 0 # hmmm, see comment in read() - - def reset_input_buffer(self): - """Flush input buffer, discarding all it’s contents.""" - if not self._isOpen: - raise portNotOpenError - - def reset_output_buffer(self): - """Clear output buffer. - - Clear output buffer, aborting the current output and discarding - all that is in the buffer. - """ - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored flushOutput') - - def send_break(self, duration=0.25): - """Send break condition. Timed, returns to idle state after given - duration.""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored sendBreak({!r})'.format(duration)) - - @property - def break_condition(self): - """Get break_condition.""" - return self.__break_condition - - @break_condition.setter - def break_condition(self, value): - """Set break_condition: Controls TXD. When active, to transmitting is - possible.""" - self.__break_condition = value - if self.logger: - self.logger.info('ignored setBreak({!r})'.format(level)) - - def setRTS(self, level=True): - """Set terminal status line: Request To Send""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored setRTS({!r})'.format(level)) - - def setDTR(self, level=True): - """Set terminal status line: Data Terminal Ready""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('ignored setDTR({!r})'.format(level)) - - def getCTS(self): - """Read terminal status line: Clear To Send""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('returning dummy for getCTS()') - return True - - def getDSR(self): - """Read terminal status line: Data Set Ready""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('returning dummy for getDSR()') - return True - - def getRI(self): - """Read terminal status line: Ring Indicator""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('returning dummy for getRI()') - return False - - def getCD(self): - """Read terminal status line: Carrier Detect""" - if not self._isOpen: - raise portNotOpenError - if self.logger: - self.logger.info('returning dummy for getCD()') - return True diff --git a/pocs/tests/test_rs232.py b/pocs/tests/test_rs232.py index 281956735..ec4adec7c 100644 --- a/pocs/tests/test_rs232.py +++ b/pocs/tests/test_rs232.py @@ -1,79 +1,171 @@ import pytest -import serial # PySerial, from https://github.com/pyserial/pyserial +import serial from pocs.utils import rs232 from pocs.utils.config import load_config +from pocs.tests.serial_handlers import NoOpSerial from pocs.tests.serial_handlers import protocol_buffers from pocs.tests.serial_handlers import protocol_no_op -from pocs.tests.serial_handlers import protocol_test +from pocs.tests.serial_handlers import protocol_hooked -THE_HOOKS = None - -def create_hooks(ser, *args, **kwargs): - THE_HOOKS = SimpleSerialHooks(ser) +def test_detect_uninstalled_scheme(): + """If our handlers aren't installed, will detect unknown scheme.""" + with pytest.raises(ValueError): + rs232.SerialData(port='no_op://') -@pytest.fixture(scope='function') +@pytest.fixture(scope='module') def handler(): - # Install our test handlers for the duration + # Install our test handlers for the duration. serial.protocol_handler_packages.append('pocs.tests.serial_handlers') yield True - # Remove our test handlers + # Remove our test handlers. serial.protocol_handler_packages.remove('pocs.tests.serial_handlers') -def test_basic(handler): +def test_detect_bogus_scheme(handler): + """When our handlers are installed, will still detect unknown scheme.""" + with pytest.raises(ValueError): + rs232.SerialData(port='bogus://') + + +@pytest.fixture(scope="function", params=[False, True]) +def threaded(request): + yield request.param + + +def test_basic_no_op(handler, threaded): # Confirm we can create the SerialData object. - ser = rs232.SerialData(port='no_op://', open_delay=0) - # Peek inside, it should have a PySerial instance as member ser. + ser = rs232.SerialData(port='no_op://', open_delay=0, threaded=threaded) + + # Peek inside, it should have a NoOpSerial instance as member ser. assert ser.ser - assert ser.ser.__class__.__name__ == 'NoOpSerial' - print(str(ser.ser)) + assert isinstance(ser.ser, NoOpSerial) + # Open is automatically called by SerialData. assert ser.is_connected - # Not using threading. + + # Listener not started, whether threaded or not. + assert ser.is_threaded == threaded assert not ser.is_listening - # no_op handler doesn't do any reading or writing. + # If threaded, start the listener. + if threaded: + ser.start() + assert ser.is_listening + assert ser.is_threaded + + # no_op handler doesn't do any reading, analogous to /dev/null, which + # never produces any output. assert '' == ser.read(retry_delay=0.01, retry_limit=2) - assert 0 == ser.write('') + # Assert how much is written, which unfortunately isn't consistent. + if threaded: + assert 6 == ser.write('abcdef') + else: + assert 0 == ser.write('abcdef') + + # If threaded, stop the listener. + if threaded: + assert ser.is_threaded + assert ser.is_listening + ser.stop() + assert not ser.is_listening + + # Disconnect from the serial port. + assert ser.is_connected ser.disconnect() assert not ser.is_connected assert not ser.is_listening + # Should no longer be able to read or write. + with pytest.raises(AssertionError): + ser.read(retry_delay=0.01, retry_limit=1) + with pytest.raises(AssertionError): + ser.write('a') -def test_without_handler(): - """If the handlers aren't installed, it should fail.""" - with pytest.raises(ValueError): - rs232.SerialData(port='no_op://') +def test_basic_io(handler, threaded): + protocol_buffers.ResetBuffers(b'abc\r\n') + ser = rs232.SerialData(port='buffers://', open_delay=0, threaded=threaded) + + # Peek inside, it should have a BuffersSerial instance as member ser. + assert isinstance(ser.ser, protocol_buffers.BuffersSerial) -def test_another_handler(handler): - """And even when installed, the wrong name won't work.""" - with pytest.raises(ValueError): - rs232.SerialData(port='bogus://') + # Listener not started, whether threaded or not. + assert ser.is_threaded == threaded + assert not ser.is_listening + # If threaded, start the listener. + if threaded: + ser.start() + assert ser.is_listening + assert ser.is_threaded + pytest.set_trace() -def test_unthreaded_io(handler): - pytest.set_trace() + # Can read one line, "abc\r\n", from the read buffer. + assert 'abc\r\n' == ser.read(retry_delay=0.1, retry_limit=10) + # Another read will fail, having exhausted the contents of the read buffer. + assert '' == ser.read(retry_delay=0.01, retry_limit=2) + + # Can write to the "device", the handler will accumulate the results. + assert 5 == ser.write('def\r\n') + assert 6 == ser.write('done\r\n') + + assert b'def\r\ndone\r\n' == protocol_buffers.GetWBufferValue() + + # If we add more to the read buffer, we can read again. + protocol_buffers.SetRBufferValue(b'line1\r\nline2\r\ndangle') + assert 'line1\r\n' == ser.read(retry_delay=10, retry_limit=20) + assert 'line2\r\n' == ser.read(retry_delay=10, retry_limit=20) + assert 'dangle' == ser.read(retry_delay=10, retry_limit=20) + + ser.disconnect() + assert not ser.is_connected + + +class ThreadedHandler(protocol_no_op.Serial): + pass + + +def test_threaded_io(handler): + protocol_hooked.Serial = ThreadedHandler protocol_buffers.ResetBuffers(b'abc\r\n') - ser = rs232.SerialData(port='buffers://', open_delay=0) + #pytest.set_trace() + ser = rs232.SerialData(port='hooked://', open_delay=0, threaded=False) assert ser # Peek inside, it should have a PySerial instance as member ser. assert ser.ser - assert ser.ser.__class__.__name__ == 'BuffersSerial' + assert ser.ser.__class__.__name__ == 'ThreadedHandler' print(str(ser.ser)) # Open is automatically called by SerialData. assert ser.is_connected - # Not using threading. + # Not threaded, and so listener not started. + assert not ser.is_threaded assert not ser.is_listening + + return - # no_op handler doesn't do any reading or writing. - assert 'abc' == ser.read(retry_delay=0.01, retry_limit=2) - assert 5 == ser.write(b'def\r\n') + # Can read one line, "abc\r\n", from the read buffer. + assert 'abc\r\n' == ser.read(retry_delay=10, retry_limit=20) + # Another read will fail, having exhausted the contents of the read buffer. + assert '' == ser.read(retry_delay=0.01, retry_limit=2) + + # Can write to the "device", the handler will accumulate the results. + assert 5 == ser.write('def\r\n') + assert 6 == ser.write('done\r\n') + + assert b'def\r\ndone\r\n' == protocol_buffers.GetWBufferValue() + + # If we add more to the read buffer, we can read again. + protocol_buffers.SetRBufferValue(b'line1\r\nline2\r\ndangle') + assert 'line1\r\n' == ser.read(retry_delay=10, retry_limit=20) + assert 'line2\r\n' == ser.read(retry_delay=10, retry_limit=20) + pytest.set_trace() + assert 'dangle' == ser.read(retry_delay=10, retry_limit=20) + ser.disconnect() assert not ser.is_connected diff --git a/pocs/utils/rs232.py b/pocs/utils/rs232.py index 38262a8ba..70fb7a4db 100644 --- a/pocs/utils/rs232.py +++ b/pocs/utils/rs232.py @@ -21,12 +21,13 @@ def __init__(self, baudrate=115200, threaded=True, name="serial_data", - open_delay=2.0): + open_delay=2.0, + retry_limit=5, + retry_delay=0.5): PanBase.__init__(self) self.ser = serial.serial_for_url(port, do_not_open=True) self.ser.baudrate = baudrate - self.is_threaded = threaded self.ser.bytesize = serial.EIGHTBITS self.ser.parity = serial.PARITY_NONE @@ -38,27 +39,34 @@ def __init__(self, self.ser.write_timeout = False self.name = name - self.queue = deque([], 1) + self.is_threaded = threaded + self.retry_limit = retry_limit + self.retry_delay = retry_delay + + self.queue = None self._is_listening = False self.process = None self._serial_io = None self.loop_delay = 2. + self._start_needed = False # Properties have been set to reasonable values, ready to open the port. self.ser.open() if self.is_threaded: + self.logger.debug("Using threads (multiprocessing)") + self._serial_io = TextIOWrapper( BufferedRWPair(self.ser, self.ser), newline='\r\n', encoding='ascii', line_buffering=True) - - self.logger.debug("Using threads (multiprocessing)") + self.queue = deque([], 1) self.process = Thread( target=self.receiving_function, args=(self.queue, )) self.process.daemon = True self.process.name = "PANOPTES_{}".format(name) + self._start_needed = True # TODO(jamessynge): Consider eliminating this sleep period, or making # it a configurable option. For one thing, it slows down tests! @@ -87,6 +95,7 @@ def start(self): self.logger.debug("Starting serial process: {}".format( self.process.name)) self._is_listening = True + self._start_needed = False self.process.start() def stop(self): @@ -145,6 +154,7 @@ def write(self, value): """ For now just pass the value along to serial object """ + assert not self._start_needed assert self.ser assert self.ser.isOpen() @@ -156,21 +166,28 @@ def write(self, value): return response - def read(self, retry_limit=5, retry_delay=0.5): + def read(self, retry_limit=None, retry_delay=None): """Reads next line of input using readline. If no response is given, delay for retry_delay and then try to read again. Fail after retry_limit attempts. """ + assert not self._start_needed assert self.ser assert self.ser.isOpen() + if retry_limit is None: + retry_limit = self.retry_limit + if retry_delay is None: + retry_delay = self.retry_delay + while True and retry_limit: if self.is_threaded: response_string = self._serial_io.readline() else: - response_string = self.ser.readline( - self.ser.inWaiting()).decode() + avail = self.ser.in_waiting + response = self.ser.readline(avail) + response_string = response.decode() if response_string > '': break @@ -191,6 +208,7 @@ def get_reading(self): try: if self.is_threaded: + assert not self._start_needed info = self.queue.pop() else: ts = time.strftime('%Y-%m-%dT%H:%M:%S %Z', time.gmtime()) From a6764e3c8a017d9dc01a7df1db7033a275366fd7 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Thu, 30 Nov 2017 21:46:52 -0500 Subject: [PATCH 07/12] Testing of rs232.SerialData, demonstrating how to hook (or mock) PySerial. This shows how we can develop mock serial devices for testing. --- pocs/tests/serial_handlers/protocol_hooked.py | 3 +- pocs/tests/test_rs232.py | 152 +++++++++--------- 2 files changed, 76 insertions(+), 79 deletions(-) diff --git a/pocs/tests/serial_handlers/protocol_hooked.py b/pocs/tests/serial_handlers/protocol_hooked.py index ba2a61f85..dc7d0b6b5 100644 --- a/pocs/tests/serial_handlers/protocol_hooked.py +++ b/pocs/tests/serial_handlers/protocol_hooked.py @@ -1,7 +1,7 @@ # This module enables a test to provide a handler for "hooked://..." urls # passed into serial.serial_for_url. To do so, set the value of # serial_class_for_url from your test to a function with the same API as -# ExampleSerialClassForUrl. +# ExampleSerialClassForUrl. Or assign your class to Serial. from pocs.tests.serial_handlers import NoOpSerial @@ -23,6 +23,7 @@ def ExampleSerialClassForUrl(url): """ return url, Serial + # Assign to this global variable from a test to override this default behavior. serial_class_for_url = ExampleSerialClassForUrl diff --git a/pocs/tests/test_rs232.py b/pocs/tests/test_rs232.py index ec4adec7c..3b0fa2bf4 100644 --- a/pocs/tests/test_rs232.py +++ b/pocs/tests/test_rs232.py @@ -1,3 +1,4 @@ +import io import pytest import serial @@ -31,14 +32,9 @@ def test_detect_bogus_scheme(handler): rs232.SerialData(port='bogus://') -@pytest.fixture(scope="function", params=[False, True]) -def threaded(request): - yield request.param - - -def test_basic_no_op(handler, threaded): +def test_basic_no_op(handler): # Confirm we can create the SerialData object. - ser = rs232.SerialData(port='no_op://', open_delay=0, threaded=threaded) + ser = rs232.SerialData(port='no_op://', open_delay=0, threaded=False) # Peek inside, it should have a NoOpSerial instance as member ser. assert ser.ser @@ -47,38 +43,15 @@ def test_basic_no_op(handler, threaded): # Open is automatically called by SerialData. assert ser.is_connected - # Listener not started, whether threaded or not. - assert ser.is_threaded == threaded - assert not ser.is_listening - - # If threaded, start the listener. - if threaded: - ser.start() - assert ser.is_listening - assert ser.is_threaded - # no_op handler doesn't do any reading, analogous to /dev/null, which # never produces any output. assert '' == ser.read(retry_delay=0.01, retry_limit=2) - - # Assert how much is written, which unfortunately isn't consistent. - if threaded: - assert 6 == ser.write('abcdef') - else: - assert 0 == ser.write('abcdef') - - # If threaded, stop the listener. - if threaded: - assert ser.is_threaded - assert ser.is_listening - ser.stop() - assert not ser.is_listening + assert 0 == ser.write('abcdef') # Disconnect from the serial port. assert ser.is_connected ser.disconnect() assert not ser.is_connected - assert not ser.is_listening # Should no longer be able to read or write. with pytest.raises(AssertionError): @@ -87,24 +60,13 @@ def test_basic_no_op(handler, threaded): ser.write('a') -def test_basic_io(handler, threaded): +def test_basic_io(handler): protocol_buffers.ResetBuffers(b'abc\r\n') - ser = rs232.SerialData(port='buffers://', open_delay=0, threaded=threaded) + ser = rs232.SerialData(port='buffers://', open_delay=0, threaded=False) # Peek inside, it should have a BuffersSerial instance as member ser. assert isinstance(ser.ser, protocol_buffers.BuffersSerial) - # Listener not started, whether threaded or not. - assert ser.is_threaded == threaded - assert not ser.is_listening - - # If threaded, start the listener. - if threaded: - ser.start() - assert ser.is_listening - assert ser.is_threaded - pytest.set_trace() - # Can read one line, "abc\r\n", from the read buffer. assert 'abc\r\n' == ser.read(retry_delay=0.1, retry_limit=10) # Another read will fail, having exhausted the contents of the read buffer. @@ -126,48 +88,82 @@ def test_basic_io(handler, threaded): assert not ser.is_connected -class ThreadedHandler(protocol_no_op.Serial): - pass - - -def test_threaded_io(handler): - protocol_hooked.Serial = ThreadedHandler - protocol_buffers.ResetBuffers(b'abc\r\n') +class HookedSerialHandler(NoOpSerial): + """Sources a line of text repeatedly, and sinks an infinite amount of input.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.r_buffer = io.BytesIO( + b"{'a': 12, 'b': [1, 2, 3, 4], 'c': {'d': 'message'}}\r\n") + + @property + def in_waiting(self): + """The number of input bytes available to read immediately.""" + if not self.is_open: + raise serialutil.portNotOpenError + total = len(self.r_buffer.getbuffer()) + avail = total - self.r_buffer.tell() + # If at end of the stream, reset the stream. + if avail <= 0: + self.r_buffer.seek(0) + avail = total + return avail + + def open(self): + """Open port. + + Raises: + SerialException if the port cannot be opened. + """ + self.is_open = True + + def close(self): + """Close port immediately.""" + self.is_open = False + + def read(self, size=1): + """Read until the end of self.r_buffer, then seek to beginning of self.r_buffer.""" + if not self.is_open: + raise serialutil.portNotOpenError + # If at end of the stream, reset the stream. + avail = self.in_waiting + return self.r_buffer.read(min(size, self.in_waiting)) + + def write(self, data): + """Write data to bitbucket.""" + if not self.is_open: + raise serialutil.portNotOpenError + return len(data) + + +def test_hooked_io(handler): + protocol_hooked.Serial = HookedSerialHandler #pytest.set_trace() ser = rs232.SerialData(port='hooked://', open_delay=0, threaded=False) - assert ser + # Peek inside, it should have a PySerial instance as member ser. assert ser.ser - assert ser.ser.__class__.__name__ == 'ThreadedHandler' + assert ser.ser.__class__.__name__ == 'HookedSerialHandler' print(str(ser.ser)) + # Open is automatically called by SerialData. assert ser.is_connected - # Not threaded, and so listener not started. - assert not ser.is_threaded - assert not ser.is_listening - - return - # Can read one line, "abc\r\n", from the read buffer. - assert 'abc\r\n' == ser.read(retry_delay=10, retry_limit=20) - # Another read will fail, having exhausted the contents of the read buffer. - assert '' == ser.read(retry_delay=0.01, retry_limit=2) - - # Can write to the "device", the handler will accumulate the results. - assert 5 == ser.write('def\r\n') - assert 6 == ser.write('done\r\n') - - assert b'def\r\ndone\r\n' == protocol_buffers.GetWBufferValue() - - # If we add more to the read buffer, we can read again. - protocol_buffers.SetRBufferValue(b'line1\r\nline2\r\ndangle') - assert 'line1\r\n' == ser.read(retry_delay=10, retry_limit=20) - assert 'line2\r\n' == ser.read(retry_delay=10, retry_limit=20) - pytest.set_trace() - assert 'dangle' == ser.read(retry_delay=10, retry_limit=20) - + # Can read many identical lines from ser. + first_line = None + for n in range(20): + line = ser.read(retry_delay=10, retry_limit=20) + if first_line: + assert line == first_line + else: + first_line = line + assert 'message' in line + + # Can write to the "device" many times. + line = 'abcdefghijklmnop' * 30 + line = line + '\r\n' + for n in range(20): + assert len(line) == ser.write(line) ser.disconnect() assert not ser.is_connected - assert not ser.is_listening - From c6349af03bb32fc9c2b8ac735643e8216a857f93 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Thu, 30 Nov 2017 21:56:54 -0500 Subject: [PATCH 08/12] Fix formatting. --- pocs/tests/serial_handlers/__init__.py | 12 ++++++------ pocs/tests/test_rs232.py | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pocs/tests/serial_handlers/__init__.py b/pocs/tests/serial_handlers/__init__.py index 0d221e7ea..9cac3ab59 100644 --- a/pocs/tests/serial_handlers/__init__.py +++ b/pocs/tests/serial_handlers/__init__.py @@ -28,7 +28,7 @@ def in_waiting(self): def open(self): """Open port. - + Raises: SerialException if the port cannot be opened. """ @@ -40,7 +40,7 @@ def close(self): def read(self, size=1): """Read size bytes. - + If a timeout is set it may return fewer characters than requested. With no timeout it will block until the requested number of bytes is read. @@ -75,7 +75,7 @@ def write(self, data): raise serialutil.portNotOpenError return 0 - #--------------------------------------------------------------------------- + # -------------------------------------------------------------------------- # There are a number of methods called by SerialBase that need to be # implemented by sub-classes, assuming their calls haven't been blocked # by replacing the calling methods/properties. These are no-op @@ -91,7 +91,7 @@ def _reconfigure_port(self): def _update_rts_state(self): """Handle rts being set to some value. - + "self.rts = value" has been executed, for some value. This may not have changed the value. """ @@ -99,7 +99,7 @@ def _update_rts_state(self): def _update_dtr_state(self): """Handle dtr being set to some value. - + "self.dtr = value" has been executed, for some value. This may not have changed the value. """ @@ -107,7 +107,7 @@ def _update_dtr_state(self): def _update_break_state(self): """Handle break_condition being set to some value. - + "self.break_condition = value" has been executed, for some value. This may not have changed the value. Note that break_condition is set and then cleared by send_break(). diff --git a/pocs/tests/test_rs232.py b/pocs/tests/test_rs232.py index 3b0fa2bf4..86608b4b0 100644 --- a/pocs/tests/test_rs232.py +++ b/pocs/tests/test_rs232.py @@ -138,7 +138,6 @@ def write(self, data): def test_hooked_io(handler): protocol_hooked.Serial = HookedSerialHandler - #pytest.set_trace() ser = rs232.SerialData(port='hooked://', open_delay=0, threaded=False) # Peek inside, it should have a PySerial instance as member ser. From 37a6be2dcddf45855a3bbce97fcd07de7ebed8b1 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Fri, 1 Dec 2017 21:02:43 -0500 Subject: [PATCH 09/12] Add some clarifying comments. --- pocs/utils/rs232.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pocs/utils/rs232.py b/pocs/utils/rs232.py index 70fb7a4db..a75181ced 100644 --- a/pocs/utils/rs232.py +++ b/pocs/utils/rs232.py @@ -222,12 +222,23 @@ def clear_buffer(self): """ Clear Response Buffer """ # Not worrying here about bytes arriving between reading in_waiting # and calling reset_input_buffer(). + # Note that Wilfred reports that the Arduino input can seriously lag behind + # realtime (e.g. 10 seconds), and that clear_buffer may exist for that reason + # (i.e. toss out any buffered input, and then read the next full line... + # which likely requires tossing out a fragment of a line). count = self.ser.in_waiting self.ser.reset_input_buffer() # self.logger.debug('Cleared {} bytes from buffer'.format(count)) def __del__(self): + """Close the serial device on delete. + + This is to avoid leaving a file or device open if there are multiple references + to the serial.Serial object. + """ try: + # If an exception is thrown when running __init__, then self.ser may not have + # been set, in which case reading that field will generate a NameError. ser = self.ser except NameError: return From 86e7307befd646bd46de3a62910f3dda673795b3 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Sat, 2 Dec 2017 20:22:18 -0500 Subject: [PATCH 10/12] Fix whitespace and comments. --- pocs/utils/rs232.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pocs/utils/rs232.py b/pocs/utils/rs232.py index 592f2023a..05e5fe798 100644 --- a/pocs/utils/rs232.py +++ b/pocs/utils/rs232.py @@ -33,6 +33,9 @@ def __init__(self, the device. threaded: Obsolete, ignored. name: Name of this object. + open_delay: Seconds to wait after opening the port. + retry_limit: Number of times to try readline() calls in read(). + retry_delay: Delay between readline() calls in read(). """ PanBase.__init__(self) @@ -78,8 +81,9 @@ def is_connected(self): return connected def connect(self): - """ Actually set up the Thread and connect to serial """ - + """Actually set up the Thread and connect to serial.""" + # TODO(jamessynge): Determine if we need this since the serial port is opened + # when the instance is created. self.logger.debug('Serial connect called') if not self.ser.isOpen(): try: @@ -95,7 +99,7 @@ def connect(self): return self.ser.isOpen() def disconnect(self): - """Closes the serial connection + """Closes the serial connection. Returns: bool: Indicates if closed or not @@ -104,9 +108,7 @@ def disconnect(self): return not self.is_connected def write(self, value): - """ - For now just pass the value along to serial object - """ + """Write value (a string) after encoding as bytes.""" assert self.ser assert self.ser.isOpen() @@ -121,7 +123,6 @@ def read(self, retry_limit=None, retry_delay=None): If no response is given, delay for retry_delay and then try to read again. Fail after retry_limit attempts. """ - assert not self._start_needed assert self.ser assert self.ser.isOpen() @@ -134,20 +135,21 @@ def read(self, retry_limit=None, retry_delay=None): response_string = self.ser.readline(self.ser.inWaiting()).decode() if response_string > '': break - time.sleep(delay) + time.sleep(retry_delay) retry_limit -= 1 # self.logger.debug('Serial read: {}'.format(response_string)) - return response_string def get_reading(self): - """ Get reading from the queue + """Read a line and return the timestamp of the read. Returns: str: Item in queue """ + # TODO(jamessynge): Consider collecting the time (utc?) after the read completes, + # so that long delays reading don't appear to have happened much earlier. try: ts = time.strftime('%Y-%m-%dT%H:%M:%S %Z', time.gmtime()) info = (ts, self.read()) @@ -170,7 +172,7 @@ def clear_buffer(self): def __del__(self): """Close the serial device on delete. - + This is to avoid leaving a file or device open if there are multiple references to the serial.Serial object. """ From 9df9bb77cfd408719a611425400657aa53f540f2 Mon Sep 17 00:00:00 2001 From: jamessynge Date: Sun, 3 Dec 2017 08:56:00 -0500 Subject: [PATCH 11/12] Applying requested review changes. --- Changelog.md | 9 ++++++++- pocs/utils/rs232.py | 18 +++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/Changelog.md b/Changelog.md index db3772138..81de41559 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,12 @@ ## [Unreleased] +- PR#164 (no sha yet) + - rs232.SerialData doesn't hide exceptions during opening of the port. + - Support added for testing of serial devices. + +- PR#148 (a436f2a127af43b6655410321f820d7660a0fe51) + - Remove threading support from rs232.SerialData, it wasn't used. + ## [0.5.1] - 2017-12-02 ### Added - First real release! @@ -11,4 +18,4 @@ - Relies on separate repositories PEAS and PACE - Automated testing with travis-ci.org - Code coverage via codecov.io -- Basic install scripts \ No newline at end of file +- Basic install scripts diff --git a/pocs/utils/rs232.py b/pocs/utils/rs232.py index 05e5fe798..4d1a729ad 100644 --- a/pocs/utils/rs232.py +++ b/pocs/utils/rs232.py @@ -62,8 +62,6 @@ def __init__(self, # Properties have been set to reasonable values, ready to open the port. self.ser.open() - # TODO(jamessynge): Consider eliminating this sleep period, or making - # it an option in the configure file. For one thing, it slows down tests! open_delay = max(0.0, float(open_delay)) self.logger.debug( 'Serial connection set up to %r, sleeping for %r seconds', self.name, open_delay) @@ -159,16 +157,14 @@ def get_reading(self): return info def clear_buffer(self): - """ Clear Response Buffer """ - # Not worrying here about bytes arriving between reading in_waiting - # and calling reset_input_buffer(). - # Note that Wilfred reports that the Arduino input can seriously lag behind - # realtime (e.g. 10 seconds), and that clear_buffer may exist for that reason - # (i.e. toss out any buffered input, and then read the next full line... - # which likely requires tossing out a fragment of a line). - count = self.ser.in_waiting + """Clear buffered data from connected port/device. + + Note that Wilfred reports that the input from an Arduino can seriously lag behind + realtime (e.g. 10 seconds), and that clear_buffer may exist for that reason (i.e. toss + out any buffered input from a device, and then read the next full line, which likely + requires tossing out a fragment of a line). + """ self.ser.reset_input_buffer() - # self.logger.debug('Cleared {} bytes from buffer'.format(count)) def __del__(self): """Close the serial device on delete. From 1fd8969e5c140e14a0c020833e7129678ee2a88a Mon Sep 17 00:00:00 2001 From: James Synge Date: Sun, 3 Dec 2017 19:36:47 -0500 Subject: [PATCH 12/12] Update Changelog.md --- Changelog.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Changelog.md b/Changelog.md index 81de41559..e376f9855 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,11 +1,7 @@ ## [Unreleased] -- PR#164 (no sha yet) - - rs232.SerialData doesn't hide exceptions during opening of the port. - - Support added for testing of serial devices. - -- PR#148 (a436f2a127af43b6655410321f820d7660a0fe51) - - Remove threading support from rs232.SerialData, it wasn't used. +- Support added for testing of serial devices. +- Remove threading support from rs232.SerialData. ## [0.5.1] - 2017-12-02 ### Added