diff --git a/Changelog.md b/Changelog.md index db3772138..e376f9855 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,8 @@ ## [Unreleased] +- Support added for testing of serial devices. +- Remove threading support from rs232.SerialData. + ## [0.5.1] - 2017-12-02 ### Added - First real release! @@ -11,4 +14,4 @@ - Relies on separate repositories PEAS and PACE - Automated testing with travis-ci.org - Code coverage via codecov.io -- Basic install scripts \ No newline at end of file +- Basic install scripts diff --git a/pocs/tests/serial_handlers/__init__.py b/pocs/tests/serial_handlers/__init__.py new file mode 100644 index 000000000..9cac3ab59 --- /dev/null +++ b/pocs/tests/serial_handlers/__init__.py @@ -0,0 +1,115 @@ +# The protocol_*.py files in this package are based on PySerial's file +# test/handlers/protocol_test.py, modified for different behaviors. +# The call serial.serial_for_url("XYZ://") looks for a class Serial in a +# file named protocol_XYZ.py in this package (i.e. directory). + +print("Importing serial_handlers __init__.py") +print("__name__:", __name__) +print("__file__:", __file__) + +from serial import serialutil + + +class NoOpSerial(serialutil.SerialBase): + """No-op implementation of PySerial's SerialBase. + + Provides no-op implementation of various methods that SerialBase expects + to have implemented by the sub-class. Can be used as is for a /dev/null + type of behavior. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @property + def in_waiting(self): + """The number of input bytes available to read immediately.""" + return 0 + + def open(self): + """Open port. + + Raises: + SerialException if the port cannot be opened. + """ + self.is_open = True + + def close(self): + """Close port immediately.""" + self.is_open = False + + def read(self, size=1): + """Read size bytes. + + If a timeout is set it may return fewer characters than requested. + With no timeout it will block until the requested number of bytes + is read. + + Args: + size: Number of bytes to read. + + Returns: + Bytes read from the port, of type 'bytes'. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + if not self.is_open: + raise serialutil.portNotOpenError + return bytes() + + def write(self, data): + """ + Args: + data: The data to write. + + Returns: + Number of bytes written. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + if not self.is_open: + raise serialutil.portNotOpenError + return 0 + + # -------------------------------------------------------------------------- + # There are a number of methods called by SerialBase that need to be + # implemented by sub-classes, assuming their calls haven't been blocked + # by replacing the calling methods/properties. These are no-op + # implementations. + + def _reconfigure_port(self): + """Reconfigure the open port after a property has been changed. + + If you need to know which property has been changed, override the + setter for the appropriate properties. + """ + pass + + def _update_rts_state(self): + """Handle rts being set to some value. + + "self.rts = value" has been executed, for some value. This may not + have changed the value. + """ + pass + + def _update_dtr_state(self): + """Handle dtr being set to some value. + + "self.dtr = value" has been executed, for some value. This may not + have changed the value. + """ + pass + + def _update_break_state(self): + """Handle break_condition being set to some value. + + "self.break_condition = value" has been executed, for some value. + This may not have changed the value. + Note that break_condition is set and then cleared by send_break(). + """ + pass diff --git a/pocs/tests/serial_handlers/protocol_buffers.py b/pocs/tests/serial_handlers/protocol_buffers.py new file mode 100644 index 000000000..d9211df57 --- /dev/null +++ b/pocs/tests/serial_handlers/protocol_buffers.py @@ -0,0 +1,105 @@ +# This module implements a handler for serial_for_url("buffers://"). + +print("Importing protocol_buffers.py") +print("__name__:", __name__) +print("__file__:", __file__) + +from pocs.tests.serial_handlers import NoOpSerial + +import io +import threading + +# r_buffer and w_buffer are binary I/O buffers. read(size=N) on an instance +# of Serial reads the next N bytes from r_buffer, and write(data) appends the +# bytes of data to w_buffer. +# NOTE: The caller (a test) is responsible for resetting buffers before tests. +_r_buffer = None +_w_buffer = None + +# The above I/O buffers are not thread safe, so we need to lock them during +# access. +_r_lock = threading.Lock() +_w_lock = threading.Lock() + + +def ResetBuffers(read_data=None): + SetRBufferValue(read_data) + with _w_lock: + global _w_buffer + _w_buffer = io.BytesIO() + + +def SetRBufferValue(data): + """Sets the r buffer to data (a bytes object).""" + if data and not isinstance(data, (bytes, bytearray)): + raise TypeError("data must by a bytes or bytearray object.") + with _r_lock: + global _r_buffer + _r_buffer = io.BytesIO(data) + + +def GetWBufferValue(): + """Returns an immutable bytes object with the value of the w buffer.""" + with _w_lock: + if _w_buffer: + return _w_buffer.getvalue() + + +class BuffersSerial(NoOpSerial): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + @property + def in_waiting(self): + if not self.is_open: + raise serialutil.portNotOpenError + with _r_lock: + return len(_r_buffer.getbuffer()) - _r_buffer.tell() + + def read(self, size=1): + """Read size bytes. + + If a timeout is set it may return fewer characters than requested. + With no timeout it will block until the requested number of bytes + is read. + + Args: + size: Number of bytes to read. + + Returns: + Bytes read from the port, of type 'bytes'. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + if not self.is_open: + raise serialutil.portNotOpenError + with _r_lock: + # TODO(jamessynge): Figure out whether and how to handle timeout. + # We might choose to generate a timeout if the caller asks for data + # beyond the end of the buffer; or simply return what is left, + # including nothing (i.e. bytes()) if there is nothing left. + return _r_buffer.read(size) + + def write(self, data): + """ + Args: + data: The data to write. + + Returns: + Number of bytes written. + + Raises: + SerialTimeoutException: In case a write timeout is configured for + the port and the time is exceeded. + """ + if not isinstance(data, (bytes, bytearray)): + raise TypeError("data must by a bytes or bytearray object.") + if not self.is_open: + raise serialutil.portNotOpenError + with _w_lock: + return _w_buffer.write(data) + + +Serial = BuffersSerial diff --git a/pocs/tests/serial_handlers/protocol_hooked.py b/pocs/tests/serial_handlers/protocol_hooked.py new file mode 100644 index 000000000..dc7d0b6b5 --- /dev/null +++ b/pocs/tests/serial_handlers/protocol_hooked.py @@ -0,0 +1,31 @@ +# This module enables a test to provide a handler for "hooked://..." urls +# passed into serial.serial_for_url. To do so, set the value of +# serial_class_for_url from your test to a function with the same API as +# ExampleSerialClassForUrl. Or assign your class to Serial. + +from pocs.tests.serial_handlers import NoOpSerial + + +def ExampleSerialClassForUrl(url): + """Implementation of serial_class_for_url called by serial.serial_for_url. + + Returns the url, possibly modified, and a factory function to be called to + create an instance of a SerialBase sub-class (or at least behaves like it). + You can return a class as that factory function, as calling a class creates + an instance of that class. + + serial.serial_for_url will call that factory function with None as the + port parameter (the first), and after creating the instance will assign + the url to the port property of the instance. + + Returns: + A tuple (url, factory). + """ + return url, Serial + + +# Assign to this global variable from a test to override this default behavior. +serial_class_for_url = ExampleSerialClassForUrl + +# Or assign your own class to this global variable. +Serial = NoOpSerial diff --git a/pocs/tests/serial_handlers/protocol_no_op.py b/pocs/tests/serial_handlers/protocol_no_op.py new file mode 100644 index 000000000..4af6c9396 --- /dev/null +++ b/pocs/tests/serial_handlers/protocol_no_op.py @@ -0,0 +1,6 @@ +# This module implements a handler for serial_for_url("no_op://"). + +from pocs.tests.serial_handlers import NoOpSerial + +# Export it as Serial so that it will be picked up by PySerial's serial_for_url. +Serial = NoOpSerial diff --git a/pocs/tests/test_rs232.py b/pocs/tests/test_rs232.py new file mode 100644 index 000000000..86608b4b0 --- /dev/null +++ b/pocs/tests/test_rs232.py @@ -0,0 +1,168 @@ +import io +import pytest +import serial + +from pocs.utils import rs232 +from pocs.utils.config import load_config + +from pocs.tests.serial_handlers import NoOpSerial +from pocs.tests.serial_handlers import protocol_buffers +from pocs.tests.serial_handlers import protocol_no_op +from pocs.tests.serial_handlers import protocol_hooked + + +def test_detect_uninstalled_scheme(): + """If our handlers aren't installed, will detect unknown scheme.""" + with pytest.raises(ValueError): + rs232.SerialData(port='no_op://') + + +@pytest.fixture(scope='module') +def handler(): + # Install our test handlers for the duration. + serial.protocol_handler_packages.append('pocs.tests.serial_handlers') + yield True + # Remove our test handlers. + serial.protocol_handler_packages.remove('pocs.tests.serial_handlers') + + +def test_detect_bogus_scheme(handler): + """When our handlers are installed, will still detect unknown scheme.""" + with pytest.raises(ValueError): + rs232.SerialData(port='bogus://') + + +def test_basic_no_op(handler): + # Confirm we can create the SerialData object. + ser = rs232.SerialData(port='no_op://', open_delay=0, threaded=False) + + # Peek inside, it should have a NoOpSerial instance as member ser. + assert ser.ser + assert isinstance(ser.ser, NoOpSerial) + + # Open is automatically called by SerialData. + assert ser.is_connected + + # no_op handler doesn't do any reading, analogous to /dev/null, which + # never produces any output. + assert '' == ser.read(retry_delay=0.01, retry_limit=2) + assert 0 == ser.write('abcdef') + + # Disconnect from the serial port. + assert ser.is_connected + ser.disconnect() + assert not ser.is_connected + + # Should no longer be able to read or write. + with pytest.raises(AssertionError): + ser.read(retry_delay=0.01, retry_limit=1) + with pytest.raises(AssertionError): + ser.write('a') + + +def test_basic_io(handler): + protocol_buffers.ResetBuffers(b'abc\r\n') + ser = rs232.SerialData(port='buffers://', open_delay=0, threaded=False) + + # Peek inside, it should have a BuffersSerial instance as member ser. + assert isinstance(ser.ser, protocol_buffers.BuffersSerial) + + # Can read one line, "abc\r\n", from the read buffer. + assert 'abc\r\n' == ser.read(retry_delay=0.1, retry_limit=10) + # Another read will fail, having exhausted the contents of the read buffer. + assert '' == ser.read(retry_delay=0.01, retry_limit=2) + + # Can write to the "device", the handler will accumulate the results. + assert 5 == ser.write('def\r\n') + assert 6 == ser.write('done\r\n') + + assert b'def\r\ndone\r\n' == protocol_buffers.GetWBufferValue() + + # If we add more to the read buffer, we can read again. + protocol_buffers.SetRBufferValue(b'line1\r\nline2\r\ndangle') + assert 'line1\r\n' == ser.read(retry_delay=10, retry_limit=20) + assert 'line2\r\n' == ser.read(retry_delay=10, retry_limit=20) + assert 'dangle' == ser.read(retry_delay=10, retry_limit=20) + + ser.disconnect() + assert not ser.is_connected + + +class HookedSerialHandler(NoOpSerial): + """Sources a line of text repeatedly, and sinks an infinite amount of input.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.r_buffer = io.BytesIO( + b"{'a': 12, 'b': [1, 2, 3, 4], 'c': {'d': 'message'}}\r\n") + + @property + def in_waiting(self): + """The number of input bytes available to read immediately.""" + if not self.is_open: + raise serialutil.portNotOpenError + total = len(self.r_buffer.getbuffer()) + avail = total - self.r_buffer.tell() + # If at end of the stream, reset the stream. + if avail <= 0: + self.r_buffer.seek(0) + avail = total + return avail + + def open(self): + """Open port. + + Raises: + SerialException if the port cannot be opened. + """ + self.is_open = True + + def close(self): + """Close port immediately.""" + self.is_open = False + + def read(self, size=1): + """Read until the end of self.r_buffer, then seek to beginning of self.r_buffer.""" + if not self.is_open: + raise serialutil.portNotOpenError + # If at end of the stream, reset the stream. + avail = self.in_waiting + return self.r_buffer.read(min(size, self.in_waiting)) + + def write(self, data): + """Write data to bitbucket.""" + if not self.is_open: + raise serialutil.portNotOpenError + return len(data) + + +def test_hooked_io(handler): + protocol_hooked.Serial = HookedSerialHandler + ser = rs232.SerialData(port='hooked://', open_delay=0, threaded=False) + + # Peek inside, it should have a PySerial instance as member ser. + assert ser.ser + assert ser.ser.__class__.__name__ == 'HookedSerialHandler' + print(str(ser.ser)) + + # Open is automatically called by SerialData. + assert ser.is_connected + + # Can read many identical lines from ser. + first_line = None + for n in range(20): + line = ser.read(retry_delay=10, retry_limit=20) + if first_line: + assert line == first_line + else: + first_line = line + assert 'message' in line + + # Can write to the "device" many times. + line = 'abcdefghijklmnop' * 30 + line = line + '\r\n' + for n in range(20): + assert len(line) == ser.write(line) + + ser.disconnect() + assert not ser.is_connected diff --git a/pocs/utils/rs232.py b/pocs/utils/rs232.py index 3d7d379de..4d1a729ad 100644 --- a/pocs/utils/rs232.py +++ b/pocs/utils/rs232.py @@ -20,7 +20,10 @@ def __init__(self, port=None, baudrate=115200, threaded=None, - name="serial_data"): + name="serial_data", + open_delay=2.0, + retry_limit=5, + retry_delay=0.5): """Init a SerialData instance. Args: @@ -30,41 +33,45 @@ def __init__(self, the device. threaded: Obsolete, ignored. name: Name of this object. + open_delay: Seconds to wait after opening the port. + retry_limit: Number of times to try readline() calls in read(). + retry_delay: Delay between readline() calls in read(). """ PanBase.__init__(self) - try: - self.ser = serial.Serial() - self.ser.port = port - self.ser.baudrate = baudrate - - self.ser.bytesize = serial.EIGHTBITS - self.ser.parity = serial.PARITY_NONE - self.ser.stopbits = serial.STOPBITS_ONE - self.ser.timeout = 1.0 - self.ser.xonxoff = False - self.ser.rtscts = False - self.ser.dsrdtr = False - self.ser.write_timeout = False - self.ser.open() - - self.name = name - - self.logger.debug( - 'Serial connection set up to {}, sleeping for two seconds'.format( - self.name)) - time.sleep(2) - self.logger.debug('SerialData created') - except Exception as err: - self.ser = None - self.logger.critical('Could not set up serial port {} {}'.format( - port, err)) + if not port: + raise ValueError('Must specify port for SerialData') + + self.name = name + self.retry_limit = retry_limit + self.retry_delay = retry_delay + + self.ser = serial.serial_for_url(port, do_not_open=True) + + # Configure the PySerial class. + self.ser.baudrate = baudrate + self.ser.bytesize = serial.EIGHTBITS + self.ser.parity = serial.PARITY_NONE + self.ser.stopbits = serial.STOPBITS_ONE + self.ser.timeout = 1.0 + self.ser.xonxoff = False + self.ser.rtscts = False + self.ser.dsrdtr = False + self.ser.write_timeout = False + + # Properties have been set to reasonable values, ready to open the port. + self.ser.open() + + open_delay = max(0.0, float(open_delay)) + self.logger.debug( + 'Serial connection set up to %r, sleeping for %r seconds', self.name, open_delay) + if open_delay > 0.0: + time.sleep(open_delay) + self.logger.debug('SerialData created') @property def is_connected(self): - """ - Checks the serial connection on the mount to determine if connection is open - """ + """True if serial port is open, False otherwise.""" connected = False if self.ser: connected = self.ser.isOpen() @@ -72,8 +79,9 @@ def is_connected(self): return connected def connect(self): - """ Actually set up the Thread and connect to serial """ - + """Actually set up the Thread and connect to serial.""" + # TODO(jamessynge): Determine if we need this since the serial port is opened + # when the instance is created. self.logger.debug('Serial connect called') if not self.ser.isOpen(): try: @@ -89,7 +97,7 @@ def connect(self): return self.ser.isOpen() def disconnect(self): - """Closes the serial connection + """Closes the serial connection. Returns: bool: Indicates if closed or not @@ -98,9 +106,7 @@ def disconnect(self): return not self.is_connected def write(self, value): - """ - For now just pass the value along to serial object - """ + """Write value (a string) after encoding as bytes.""" assert self.ser assert self.ser.isOpen() @@ -109,35 +115,39 @@ def write(self, value): return response - def read(self): - """ - Reads value using readline - If no response is given, delay and then try to read again. Fail after 10 attempts + def read(self, retry_limit=None, retry_delay=None): + """Reads next line of input using readline. + + If no response is given, delay for retry_delay and then try to read + again. Fail after retry_limit attempts. """ assert self.ser assert self.ser.isOpen() - retry_limit = 5 - delay = 0.5 + if retry_limit is None: + retry_limit = self.retry_limit + if retry_delay is None: + retry_delay = self.retry_delay while True and retry_limit: response_string = self.ser.readline(self.ser.inWaiting()).decode() if response_string > '': break - time.sleep(delay) + time.sleep(retry_delay) retry_limit -= 1 # self.logger.debug('Serial read: {}'.format(response_string)) - return response_string def get_reading(self): - """ Get reading from the queue + """Read a line and return the timestamp of the read. Returns: str: Item in queue """ + # TODO(jamessynge): Consider collecting the time (utc?) after the read completes, + # so that long delays reading don't appear to have happened much earlier. try: ts = time.strftime('%Y-%m-%dT%H:%M:%S %Z', time.gmtime()) info = (ts, self.read()) @@ -147,14 +157,26 @@ def get_reading(self): return info def clear_buffer(self): - """ Clear Response Buffer """ - count = 0 - while self.ser.inWaiting() > 0: - count += 1 - self.ser.read(1) + """Clear buffered data from connected port/device. - # self.logger.debug('Cleared {} bytes from buffer'.format(count)) + Note that Wilfred reports that the input from an Arduino can seriously lag behind + realtime (e.g. 10 seconds), and that clear_buffer may exist for that reason (i.e. toss + out any buffered input from a device, and then read the next full line, which likely + requires tossing out a fragment of a line). + """ + self.ser.reset_input_buffer() def __del__(self): - if self.ser: + """Close the serial device on delete. + + This is to avoid leaving a file or device open if there are multiple references + to the serial.Serial object. + """ + try: + # If an exception is thrown when running __init__, then self.ser may not have + # been set, in which case reading that field will generate a NameError. + ser = self.ser + except NameError: + return + if ser: self.ser.close()