Make auto interpretation work in live mode during simulation (#583)

* Update adaptive noise calculation -> increase performance
* Add automatic center to simulation rx dialog

data/azure-pipelines.yml

@@ -34,13 +34,13 @@ jobs:
 
   - script: |
       touch tests/show_gui
-      pytest -s --boxed --junitxml=junit/test-results.xml --cov=src/urh --cov-report=xml --cov-report=html --cov-config tests/.coveragerc tests
+      pytest --boxed --junitxml=junit/test-results.xml --cov=src/urh --cov-report=xml --cov-report=html --cov-config tests/.coveragerc tests
     displayName: 'Run pytest with coverage'
     condition: eq(variables['python.version'], '3.6')
 
   - script: |
       touch tests/show_gui
-      pytest -s --boxed --junitxml=junit/test-results.xml tests
+      pytest --boxed --junitxml=junit/test-results.xml tests
     displayName: 'Run pytest without coverage'
     condition: ne(variables['python.version'], '3.6')
 
@@ -162,7 +162,7 @@ jobs:
       assetUploadMode: 'replace'
       addChangeLog: true
 
-  - script: pytest --junitxml=junit/test-results.xml -s -v tests
+  - script: pytest --junitxml=junit/test-results.xml tests
     displayName: 'Run pytest'
 
   - task: PublishTestResults@2
@@ -202,7 +202,7 @@ jobs:
   - script: python setup.py build_ext --inplace
     displayName: "Build extensions"
 
-  - script: pytest --junitxml=junit/test-results.xml -s -v tests
+  - script: pytest --junitxml=junit/test-results.xml tests
     displayName: 'Run pytest'
 
   - task: PublishTestResults@2

data/ui/send_recv_sniff_settings.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>482</width>
-    <height>377</height>
+    <height>388</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -147,19 +147,6 @@ QGroupBox::indicator:checked {
            </property>
           </widget>
          </item>
-         <item row="2" column="1">
-          <widget class="QDoubleSpinBox" name="spinbox_sniff_Center">
-           <property name="decimals">
-            <number>4</number>
-           </property>
-           <property name="minimum">
-            <double>-3.140000000000000</double>
-           </property>
-           <property name="maximum">
-            <double>3.140000000000000</double>
-           </property>
-          </widget>
-         </item>
          <item row="3" column="0">
           <widget class="QLabel" name="label_sniff_BitLength">
            <property name="text">
@@ -303,6 +290,36 @@ QGroupBox::indicator:checked {
            </property>
           </widget>
          </item>
+         <item row="2" column="1">
+          <layout class="QHBoxLayout" name="horizontalLayout_4">
+           <item>
+            <widget class="QDoubleSpinBox" name="spinbox_sniff_Center">
+             <property name="sizePolicy">
+              <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+               <horstretch>0</horstretch>
+               <verstretch>0</verstretch>
+              </sizepolicy>
+             </property>
+             <property name="decimals">
+              <number>4</number>
+             </property>
+             <property name="minimum">
+              <double>-3.140000000000000</double>
+             </property>
+             <property name="maximum">
+              <double>3.140000000000000</double>
+             </property>
+            </widget>
+           </item>
+           <item>
+            <widget class="QCheckBox" name="checkBoxAutoCenter">
+             <property name="text">
+              <string>Automatic</string>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </item>
         </layout>
        </widget>
       </item>
@@ -314,7 +331,6 @@ QGroupBox::indicator:checked {
  <tabstops>
   <tabstop>groupBoxSniffSettings</tabstop>
   <tabstop>spinbox_sniff_Noise</tabstop>
-  <tabstop>spinbox_sniff_Center</tabstop>
   <tabstop>spinbox_sniff_BitLen</tabstop>
   <tabstop>spinbox_sniff_ErrorTolerance</tabstop>
   <tabstop>combox_sniff_Modulation</tabstop>

src/urh/ainterpretation/AutoInterpretation.py

@@ -28,7 +28,6 @@ def min_without_outliers(data: np.ndarray, z=2):
 
     return np.min(data[abs(data - np.mean(data)) <= z * np.std(data)])
 
-
 def get_most_frequent_value(values: list):
     """
     Return the most frequent value in list.
@@ -192,23 +191,40 @@ def detect_modulation_for_messages(signal: np.ndarray, message_indices: list) ->
     return max(set(modulations_for_messages), key=modulations_for_messages.count)
 
 
-def detect_center(rectangular_signal: np.ndarray):
+def detect_center(rectangular_signal: np.ndarray, max_size=None):
     rect = rectangular_signal[rectangular_signal > -4]  # do not consider noise
 
+    # Ignore the first and last 5% of samples,
+    # because there tends to be an overshoot at start/end of rectangular signal
+    rect = rect[int(0.05*len(rect)):int(0.95*len(rect))]
+
+    if max_size is not None and len(rect) > max_size:
+        rect = rect[0:max_size]
+
     hist_min, hist_max = util.minmax(rect)
-    hist_step = 0.05
+
+    # The step size of histogram is set to variance of the rectangular signal
+    # If a signal has low variance we need to be more accurate at center detection
+    hist_step = float(np.var(rect))
 
     y, x = np.histogram(rect, bins=np.arange(hist_min, hist_max + hist_step, hist_step))
 
     num_values = 2
     most_common_levels = []
 
+    window_size = max(2, int(0.05*len(y)))
+
+    def get_elem(arr, index: int, default):
+        if 0 <= index < len(arr):
+            return arr[index]
+        else:
+            return default
+
     for index in np.argsort(y)[::-1]:
         # check if we have a local maximum in histogram, if yes, append the value
-        left = y[index - 1] if index > 0 else 0
-        right = y[index + 1] if index < len(y) - 1 else 0
-
-        if left < y[index] and y[index] > right:
+        if all(y[index] > get_elem(y, index+i, 0) and
+               y[index] > get_elem(y, index-i, 0)
+               for i in range(1, window_size+1)):
             most_common_levels.append(x[index])
 
         if len(most_common_levels) == num_values:

src/urh/controller/widgets/SniffSettingsWidget.py

@@ -38,6 +38,9 @@ def __init__(self, device_name: str, project_manager: ProjectManager, signal=Non
                                        backend_handler=BackendHandler() if backend_handler is None else backend_handler,
                                        network_raw_mode=network_raw_mode)
 
+        self.sniffer.adaptive_noise = self.ui.checkBoxAdaptiveNoise.isChecked()
+        self.sniffer.automatic_center = self.ui.checkBoxAutoCenter.isChecked()
+
         self.create_connects()
         self.ui.comboBox_sniff_encoding.currentIndexChanged.emit(self.ui.comboBox_sniff_encoding.currentIndex())
         self.ui.comboBox_sniff_viewtype.setCurrentIndex(constants.SETTINGS.value('default_view', 0, int))
@@ -77,6 +80,8 @@ def set_val(widget, key: str, default):
         set_val(self.ui.combox_sniff_Modulation, "modulation_index", signal.modulation_type if signal else 1)
         self.ui.comboBox_sniff_encoding.setCurrentText(conf_dict.get("decoding_name", ""))
         self.ui.checkBoxAdaptiveNoise.setChecked(bool(conf_dict.get("adaptive_noise", False)))
+        self.ui.checkBoxAutoCenter.setChecked(bool(conf_dict.get("automatic_center", False)))
+        self.ui.spinbox_sniff_Center.setDisabled(self.ui.checkBoxAutoCenter.isChecked())
 
         self.emit_editing_finished_signals()
 
@@ -92,6 +97,7 @@ def create_connects(self):
         self.ui.checkBox_sniff_Timestamp.clicked.connect(self.on_checkbox_sniff_timestamp_clicked)
         self.ui.btn_sniff_use_signal.clicked.connect(self.on_btn_sniff_use_signal_clicked)
         self.ui.checkBoxAdaptiveNoise.clicked.connect(self.on_check_box_adaptive_noise_clicked)
+        self.ui.checkBoxAutoCenter.clicked.connect(self.on_check_box_auto_center_clicked)
 
     def emit_editing_finished_signals(self):
         self.ui.spinbox_sniff_Noise.editingFinished.emit()
@@ -108,7 +114,8 @@ def emit_sniff_parameters_changed(self):
                                                 tolerance=self.sniffer.signal.tolerance,
                                                 modulation_index=self.sniffer.signal.modulation_type,
                                                 decoding_name=self.sniffer.decoder.name,
-                                                adaptive_noise=self.sniffer.adaptive_noise))
+                                                adaptive_noise=self.sniffer.adaptive_noise,
+                                                automatic_center=self.sniffer.automatic_center))
 
     @pyqtSlot()
     def on_noise_edited(self):
@@ -186,3 +193,8 @@ def on_btn_sniff_use_signal_clicked(self):
     @pyqtSlot()
     def on_check_box_adaptive_noise_clicked(self):
         self.sniffer.adaptive_noise = self.ui.checkBoxAdaptiveNoise.isChecked()
+
+    @pyqtSlot()
+    def on_check_box_auto_center_clicked(self):
+        self.sniffer.automatic_center = self.ui.checkBoxAutoCenter.isChecked()
+        self.ui.spinbox_sniff_Center.setDisabled(self.ui.checkBoxAutoCenter.isChecked())

src/urh/signalprocessing/ProtocolSniffer.py

@@ -5,8 +5,9 @@
 
 import numpy as np
 from PyQt5.QtCore import pyqtSignal, QObject
-
 from urh.cythonext.signal_functions import grab_pulse_lens
+
+from urh.ainterpretation import AutoInterpretation
 from urh.dev.BackendHandler import BackendHandler, Backends
 from urh.dev.VirtualDevice import VirtualDevice, Mode
 from urh.signalprocessing.Message import Message
@@ -24,6 +25,8 @@ class ProtocolSniffer(ProtocolAnalyzer, QObject):
     stopped = pyqtSignal()
     message_sniffed = pyqtSignal(int)
 
+    BUFFER_SIZE_MB = 100
+
     def __init__(self, bit_len: int, center: float, noise: float, tolerance: int,
                  modulation_type: int, device: str, backend_handler: BackendHandler, network_raw_mode=False):
         signal = Signal("", "LiveSignal")
@@ -45,9 +48,12 @@ def __init__(self, bit_len: int, center: float, noise: float, tolerance: int,
         self.rcv_device.started.connect(self.__emit_started)
         self.rcv_device.stopped.connect(self.__emit_stopped)
 
-        self.data_cache = []
+        self.__buffer = np.zeros(int(self.BUFFER_SIZE_MB * 1000 * 1000 / 8), dtype=np.complex64)
+        self.__current_buffer_index = 0
+
         self.reading_data = False
         self.adaptive_noise = False
+        self.automatic_center = False
 
         self.pause_length = 0
         self.is_running = False
@@ -57,6 +63,20 @@ def __init__(self, bit_len: int, center: float, noise: float, tolerance: int,
         self.__sniff_file = ""
         self.__store_data = True
 
+    def __add_to_buffer(self, data: np.ndarray):
+        n = len(data)
+        if n + self.__current_buffer_index > len(self.__buffer):
+            n = len(self.__buffer) - self.__current_buffer_index - 1
+
+        self.__buffer[self.__current_buffer_index:self.__current_buffer_index + n] = data[:]
+        self.__current_buffer_index += n
+
+    def __clear_buffer(self):
+        self.__current_buffer_index = 0
+
+    def __buffer_is_full(self):
+        return self.__current_buffer_index >= len(self.__buffer) - 2
+
     def decoded_to_string(self, view: int, start=0, include_timestamps=True):
         result = []
         for msg in self.messages[start:]:
@@ -118,7 +138,7 @@ def check_for_data(self):
                     msg = Message.from_plain_bits_str(bit_str)
                     msg.decoder = self.decoder
                     self.messages.append(msg)
-                    self.message_sniffed.emit(len(self.messages)-1)
+                    self.message_sniffed.emit(len(self.messages) - 1)
 
                 self.rcv_device.free_data()  # do not store received bits twice
 
@@ -140,30 +160,36 @@ def __demodulate_data(self, data):
         if len(data) == 0:
             return
 
-        rssi_squared = np.mean(data.real ** 2 + data.imag ** 2)
-        is_above_noise = rssi_squared > self.signal.noise_threshold ** 2
+        power_spectrum = data.real ** 2 + data.imag ** 2
+        is_above_noise = np.sqrt(np.mean(power_spectrum)) > self.signal.noise_threshold
+
         if self.adaptive_noise and not is_above_noise:
-            self.signal.noise_threshold = 0.9 * self.signal.noise_threshold + 0.1 * np.max(np.abs(data))
+            self.signal.noise_threshold = 0.9 * self.signal.noise_threshold + 0.1 * np.sqrt(np.max(power_spectrum))
 
         if is_above_noise:
-            self.data_cache.append(data)
+            self.__add_to_buffer(data)
             self.pause_length = 0
-            return
+            if not self.__buffer_is_full():
+                return
         else:
             self.pause_length += len(data)
             if self.pause_length < 10 * self.signal.bit_len:
-                self.data_cache.append(data)
-                return
+                self.__add_to_buffer(data)
+                if not self.__buffer_is_full():
+                    return
 
-        if len(self.data_cache) == 0:
+        if self.__current_buffer_index == 0:
             return
 
         # clear cache and start a new message
-        self.signal._fulldata = np.concatenate(self.data_cache)
-        self.data_cache.clear()
+        self.signal._fulldata = self.__buffer[0:self.__current_buffer_index]
+        self.__clear_buffer()
         self.signal._qad = None
 
         bit_len = self.signal.bit_len
+        if self.automatic_center:
+            self.signal.qad_center = AutoInterpretation.detect_center(self.signal.qad, max_size=150*self.signal.bit_len)
+
         ppseq = grab_pulse_lens(self.signal.qad, self.signal.qad_center,
                                 self.signal.tolerance, self.signal.modulation_type, self.signal.bit_len)
 
@@ -173,7 +199,7 @@ def __demodulate_data(self, data):
             message = Message(bits, pause, bit_len=bit_len, message_type=self.default_message_type,
                               decoder=self.decoder)
             self.messages.append(message)
-            self.message_sniffed.emit(len(self.messages)-1)
+            self.message_sniffed.emit(len(self.messages) - 1)
 
     def stop(self):
         self.is_running = False
@@ -184,7 +210,7 @@ def stop(self):
             logger.error("Sniff thread is still alive")
 
     def clear(self):
-        self.data_cache.clear()
+        self.__clear_buffer()
         self.messages.clear()
 
     def __emit_started(self):

src/urh/ui/ui_send_recv_sniff_settings.py

@@ -9,7 +9,7 @@
 class Ui_SniffSettings(object):
     def setupUi(self, SniffSettings):
         SniffSettings.setObjectName("SniffSettings")
-        SniffSettings.resize(482, 377)
+        SniffSettings.resize(482, 388)
         self.verticalLayout = QtWidgets.QVBoxLayout(SniffSettings)
         self.verticalLayout.setContentsMargins(0, 0, 0, 0)
         self.verticalLayout.setObjectName("verticalLayout")
@@ -84,12 +84,6 @@ def setupUi(self, SniffSettings):
         self.label_sniff_Center = QtWidgets.QLabel(self.frame)
         self.label_sniff_Center.setObjectName("label_sniff_Center")
         self.gridLayout.addWidget(self.label_sniff_Center, 2, 0, 1, 1)
-        self.spinbox_sniff_Center = QtWidgets.QDoubleSpinBox(self.frame)
-        self.spinbox_sniff_Center.setDecimals(4)
-        self.spinbox_sniff_Center.setMinimum(-3.14)
-        self.spinbox_sniff_Center.setMaximum(3.14)
-        self.spinbox_sniff_Center.setObjectName("spinbox_sniff_Center")
-        self.gridLayout.addWidget(self.spinbox_sniff_Center, 2, 1, 1, 1)
         self.label_sniff_BitLength = QtWidgets.QLabel(self.frame)
         self.label_sniff_BitLength.setObjectName("label_sniff_BitLength")
         self.gridLayout.addWidget(self.label_sniff_BitLength, 3, 0, 1, 1)
@@ -155,14 +149,30 @@ def setupUi(self, SniffSettings):
         self.lineEdit_sniff_OutputFile.setClearButtonEnabled(True)
         self.lineEdit_sniff_OutputFile.setObjectName("lineEdit_sniff_OutputFile")
         self.gridLayout.addWidget(self.lineEdit_sniff_OutputFile, 8, 1, 1, 1)
+        self.horizontalLayout_4 = QtWidgets.QHBoxLayout()
+        self.horizontalLayout_4.setObjectName("horizontalLayout_4")
+        self.spinbox_sniff_Center = QtWidgets.QDoubleSpinBox(self.frame)
+        sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed)
+        sizePolicy.setHorizontalStretch(0)
+        sizePolicy.setVerticalStretch(0)
+        sizePolicy.setHeightForWidth(self.spinbox_sniff_Center.sizePolicy().hasHeightForWidth())
+        self.spinbox_sniff_Center.setSizePolicy(sizePolicy)
+        self.spinbox_sniff_Center.setDecimals(4)
+        self.spinbox_sniff_Center.setMinimum(-3.14)
+        self.spinbox_sniff_Center.setMaximum(3.14)
+        self.spinbox_sniff_Center.setObjectName("spinbox_sniff_Center")
+        self.horizontalLayout_4.addWidget(self.spinbox_sniff_Center)
+        self.checkBoxAutoCenter = QtWidgets.QCheckBox(self.frame)
+        self.checkBoxAutoCenter.setObjectName("checkBoxAutoCenter")
+        self.horizontalLayout_4.addWidget(self.checkBoxAutoCenter)
+        self.gridLayout.addLayout(self.horizontalLayout_4, 2, 1, 1, 1)
         self.gridLayout_3.addWidget(self.frame, 0, 0, 1, 1)
         self.verticalLayout.addWidget(self.groupBoxSniffSettings)
 
         self.retranslateUi(SniffSettings)
         self.groupBoxSniffSettings.toggled['bool'].connect(self.frame.setVisible)
         SniffSettings.setTabOrder(self.groupBoxSniffSettings, self.spinbox_sniff_Noise)
-        SniffSettings.setTabOrder(self.spinbox_sniff_Noise, self.spinbox_sniff_Center)
-        SniffSettings.setTabOrder(self.spinbox_sniff_Center, self.spinbox_sniff_BitLen)
+        SniffSettings.setTabOrder(self.spinbox_sniff_Noise, self.spinbox_sniff_BitLen)
         SniffSettings.setTabOrder(self.spinbox_sniff_BitLen, self.spinbox_sniff_ErrorTolerance)
         SniffSettings.setTabOrder(self.spinbox_sniff_ErrorTolerance, self.combox_sniff_Modulation)
         SniffSettings.setTabOrder(self.combox_sniff_Modulation, self.comboBox_sniff_encoding)
@@ -194,5 +204,6 @@ def retranslateUi(self, SniffSettings):
         self.checkBox_sniff_Timestamp.setText(_translate("SniffSettings", "Show Timestamp"))
         self.label_sniff_OutputFile.setText(_translate("SniffSettings", "Write bitstream to file:"))
         self.lineEdit_sniff_OutputFile.setPlaceholderText(_translate("SniffSettings", "None"))
+        self.checkBoxAutoCenter.setText(_translate("SniffSettings", "Automatic"))
 
 from . import urh_rc