make modulation backend more generic (#660)

src/urh/awre/engines/LengthEngine.py

@@ -3,9 +3,9 @@
 
 import numpy as np
 
-from urh.awre.CommonRange import CommonRange, EmptyCommonRange
+from urh.awre.CommonRange import CommonRange
 from urh.awre.engines.Engine import Engine
-from urh.cythonext import awre_util
+from urh.cythonext import util
 
 
 class LengthEngine(Engine):
@@ -177,7 +177,7 @@ def choose_high_scored_ranges(self, scored_ranges: dict, bitvectors_by_n_gram_le
 
     @staticmethod
     def score_bits(bits: np.ndarray, target_length: int, position: int, byteorder="big"):
-        value = awre_util.bit_array_to_number(bits, len(bits))
+        value = util.bit_array_to_number(bits, len(bits))
         if byteorder == "little":
             if len(bits) > 8 and len(bits) % 8 == 0:
                 n = len(bits) // 8

src/urh/cli/urh_cli.py

@@ -89,7 +89,7 @@ def build_modulator_from_args(arguments: argparse.Namespace):
     result.carrier_freq_hz = float(arguments.carrier_frequency)
     result.carrier_amplitude = float(arguments.carrier_amplitude)
     result.carrier_phase_deg = float(arguments.carrier_phase)
-    result.samples_per_bit = int(arguments.bit_length)
+    result.samples_per_symbol = int(arguments.bit_length)
 
     if arguments.modulation_type == "ASK":
         if arguments.parameter_zero.endswith("%"):
@@ -222,7 +222,7 @@ def modulate_messages(messages, modulator):
         return None
 
     cli_progress_bar(0, len(messages), title="Modulating")
-    nsamples = sum(int(len(msg.encoded_bits) * modulator.samples_per_bit + msg.pause) for msg in messages)
+    nsamples = sum(int(len(msg.encoded_bits) * modulator.samples_per_symbol + msg.pause) for msg in messages)
     buffer = IQArray(None, dtype=np.float32, n=nsamples)
     pos = 0
     for i, msg in enumerate(messages):

src/urh/controller/GeneratorTabController.py

@@ -102,7 +102,7 @@ def modulators(self):
 
     @property
     def total_modulated_samples(self) -> int:
-        return sum(int(len(msg.encoded_bits) * self.__get_modulator_of_message(msg).samples_per_bit + msg.pause)
+        return sum(int(len(msg.encoded_bits) * self.__get_modulator_of_message(msg).samples_per_symbol + msg.pause)
                    for msg in self.table_model.protocol.messages)
 
     @modulators.setter
@@ -213,7 +213,7 @@ def bootstrap_modulator(self, protocol: ProtocolAnalyzer):
             return
 
         modulator = self.modulators[0]
-        modulator.samples_per_bit = protocol.messages[0].bit_len
+        modulator.samples_per_symbol = protocol.messages[0].bit_len
 
         if protocol.signal:
             modulator.sample_rate = protocol.signal.sample_rate
@@ -528,7 +528,7 @@ def refresh_estimated_time(self):
             return
 
         avg_msg_len = numpy.mean([len(msg.encoded_bits) for msg in c.messages])
-        avg_bit_len = numpy.mean([m.samples_per_bit for m in self.modulators])
+        avg_bit_len = numpy.mean([m.samples_per_symbol for m in self.modulators])
         avg_sample_rate = numpy.mean([m.sample_rate for m in self.modulators])
         pause_samples = sum(c.pauses)
         nsamples = c.num_messages * avg_msg_len * avg_bit_len + pause_samples

src/urh/controller/dialogs/ModulatorDialog.py

@@ -126,7 +126,7 @@ def set_ui_for_current_modulator(self):
         self.ui.comboBoxModulationType.setCurrentIndex(index)
         self.ui.doubleSpinBoxCarrierFreq.setValue(self.current_modulator.carrier_freq_hz)
         self.ui.doubleSpinBoxCarrierPhase.setValue(self.current_modulator.carrier_phase_deg)
-        self.ui.spinBoxBitLength.setValue(self.current_modulator.samples_per_bit)
+        self.ui.spinBoxBitLength.setValue(self.current_modulator.samples_per_symbol)
         self.ui.spinBoxSampleRate.setValue(self.current_modulator.sample_rate)
         self.ui.spinBoxParameter0.setValue(self.current_modulator.param_for_zero)
         self.ui.spinBoxParameter1.setValue(self.current_modulator.param_for_one)
@@ -374,7 +374,7 @@ def on_carrier_phase_changed(self):
 
     @pyqtSlot()
     def on_bit_len_changed(self):
-        self.current_modulator.samples_per_bit = self.ui.spinBoxBitLength.value()
+        self.current_modulator.samples_per_symbol = self.ui.spinBoxBitLength.value()
         self.draw_carrier()
         self.draw_data_bits()
         self.draw_modulated()

src/urh/controller/dialogs/SimulatorDialog.py

@@ -12,6 +12,7 @@
 from urh.controller.widgets.SniffSettingsWidget import SniffSettingsWidget
 from urh.dev.BackendHandler import BackendHandler
 from urh.dev.EndlessSender import EndlessSender
+from urh.signalprocessing.IQArray import IQArray
 from urh.simulator.Simulator import Simulator
 from urh.simulator.SimulatorConfiguration import SimulatorConfiguration
 from urh.ui.SimulatorScene import SimulatorScene
@@ -415,10 +416,12 @@ def on_checkbox_capture_full_rx_clicked(self):
     @pyqtSlot()
     def on_btn_save_rx_clicked(self):
         rx_device = self.simulator.sniffer.rcv_device
-        if isinstance(rx_device.data, np.ndarray):
-            filename = FileOperator.get_save_file_name("simulation_capture.complex")
+        if isinstance(rx_device.data, np.ndarray) or isinstance(rx_device.data, IQArray):
+            data = IQArray(rx_device.data[:rx_device.current_index])
+            filename = FileOperator.save_data_dialog("simulation_capture", data,
+                                                     sample_rate=rx_device.sample_rate, parent=self)
             if filename:
-                rx_device.data[:rx_device.current_index].tofile(filename)
+                data.tofile(filename)
                 self.rx_file_saved.emit(filename)
 
     @pyqtSlot()

src/urh/cythonext/awre_util.pyx

@@ -7,12 +7,10 @@ from libc.math cimport floor, ceil, pow
 from libc.stdlib cimport malloc, free
 
 from libcpp cimport bool
-from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, int32_t, int8_t, int64_t
-
-from array import array
-
+from libc.stdint cimport uint8_t, uint32_t, int32_t, int64_t
 
 from urh.cythonext.util import crc
+from urh.cythonext.util cimport bit_array_to_number
 
 cpdef set find_longest_common_sub_sequence_indices(np.uint8_t[::1] seq1, np.uint8_t[::1] seq2):
     cdef unsigned int i, j, longest = 0, counter = 0, len_bits1 = len(seq1), len_bits2 = len(seq2)
@@ -302,19 +300,6 @@ cpdef list find_occurrences(np.uint8_t[::1] a, np.uint8_t[::1] b,
 
     return result
 
-cpdef unsigned long long bit_array_to_number(uint8_t[::1] bits, int64_t end, int64_t start=0) nogil:
-    if end < 1:
-        return 0
-
-    cdef long long i, acc = 1
-    cdef unsigned long long result = 0
-
-    for i in range(start, end):
-        result += bits[end-1-i+start] * acc
-        acc *= 2
-
-    return result
-
 cpdef np.ndarray[np.int32_t, ndim=2, mode="c"] create_seq_number_difference_matrix(list bitvectors, int n_gram_length):
     """
     Create the difference matrix e.g.

src/urh/cythonext/signal_functions.pyx

@@ -4,11 +4,13 @@ import cython
 import numpy as np
 from libcpp cimport bool
 
-from urh.cythonext.util cimport IQ, iq
+from libc.stdint cimport uint8_t, uint16_t, uint32_t, int64_t
+from urh.cythonext.util cimport IQ, iq, bit_array_to_number
 
 from cython.parallel import prange
 from libc.math cimport atan2, sqrt, M_PI
 
+
 cdef extern from "math.h" nogil:
     float cosf(float x)
     float sinf(float x)
@@ -43,134 +45,79 @@ cdef get_numpy_dtype(iq cython_type):
     else:
         raise ValueError("dtype {} not supported for modulation".format(cython.typeof(cython_type)))
 
-cpdef modulate_fsk(unsigned char[:] bit_array, unsigned long pause, unsigned long start,
-                                      float a, float freq0, float freq1, float phi, float sample_rate,
-                                      long long samples_per_bit, iq iq_type):
-    cdef long long i = 0, j = 0, index = 0
-    cdef float t = 0, f = 0, arg = 0, f_next = 0, phase = 0
-    cdef long long total_samples = int(len(bit_array) * samples_per_bit + pause)
-
-    result = np.zeros((total_samples, 2), dtype=get_numpy_dtype(iq_type))
-    cdef iq[:, ::1] result_view = result
-    cdef float* phases = <float *>malloc(total_samples * sizeof(float))
-
-    for i in range(0, samples_per_bit):
-        phases[i] = phi
-
-    cdef long long num_bits = len(bit_array)
-    with cython.cdivision:
-        for i in range(1, num_bits):
-            phase = phases[i*samples_per_bit-1]
-
-            # We need to correct the phase on transitions between 0 and 1
-            if bit_array[i-1] != bit_array[i]:
-                t = (i*samples_per_bit+start-1) / sample_rate
-                f = freq0 if bit_array[i-1] == 0 else freq1
-                f_next = freq0 if bit_array[i] == 0 else freq1
-                phase = (phase + 2 * M_PI * t * (f - f_next)) % (2 * M_PI)
+cpdef modulate_c(uint8_t[:] bits, uint32_t samples_per_symbol, str modulation_type,
+                 float[:] parameters, uint16_t bits_per_symbol,
+                 float carrier_amplitude, float carrier_frequency, float carrier_phase, float sample_rate,
+                 uint32_t pause, uint32_t start, iq iq_type,
+                 float gauss_bt=0.5, float filter_width=1.0):
+    cdef int64_t i = 0, j = 0, index = 0, s_i = 0
+    cdef uint32_t total_symbols = int(len(bits) // bits_per_symbol)
+    cdef int64_t total_samples = total_symbols * samples_per_symbol + pause
 
-            for j in range(i*samples_per_bit, (i+1)*samples_per_bit):
-                phases[j] = phase
+    cdef float a = carrier_amplitude, f = carrier_frequency, phi = carrier_phase
 
+    cdef float t = 0, arg = 0
 
-    cdef long long loop_end = total_samples-pause
-    for i in prange(0, loop_end, nogil=True, schedule="static"):
-        t = (i+start) / sample_rate
-        index = <long long>(i/samples_per_bit)
-        f = freq0 if bit_array[index] == 0 else freq1
-
-        arg = 2 * M_PI * f * t + phases[i]
-        result_view[i, 0] = <iq>(a * cosf(arg))
-        result_view[i, 1] = <iq>(a * sinf(arg))
+    result = np.zeros((total_samples, 2), dtype=get_numpy_dtype(iq_type))
+    cdef iq[:, ::1] result_view = result
 
-        # We need to correct the phase on transitions between 0 and 1
-        # if i < loop_end - 1 and (i+1) % samples_per_bit == 0:
-        #     index = <long long>((i+1)/samples_per_bit)
-        #     f_next = freq0 if bit_array[index] == 0 else freq1
-        #     phi += 2 * M_PI * t * (f - f_next)
-        #     phi = phi % (2 * M_PI)
+    cdef bool is_fsk = modulation_type.lower() == "fsk"
+    cdef bool is_ask = modulation_type.lower() == "ask"
+    cdef bool is_psk = modulation_type.lower() == "psk"
+    cdef bool is_gfsk = modulation_type.lower() == "gfsk"
 
-    free(phases)
-    return result
+    assert is_fsk or is_ask or is_psk or is_gfsk
 
-cpdef modulate_ask(unsigned char[:] bit_array, unsigned long pause, unsigned long start,
-                            double a0, double a1, double f, double phi, double sample_rate,
-                            unsigned long samples_per_bit, iq iq_type):
-    cdef long long i = 0, index = 0
-    cdef float t = 0, a = 0, arg = 0
-    cdef long long total_samples = int(len(bit_array) * samples_per_bit + pause)
+    cdef np.ndarray[np.float32_t, ndim=2] gauss_filtered_freqs_phases
+    if is_gfsk:
+        gauss_filtered_freqs_phases = get_gauss_filtered_freqs_phases(bits, parameters, total_symbols,
+                                                                      samples_per_symbol, sample_rate, carrier_phase,
+                                                                      start, gauss_bt, filter_width)
 
-    result = np.zeros((total_samples, 2), dtype=get_numpy_dtype(iq_type))
-    cdef iq[:, ::1] result_view = result
+    for s_i in prange(0, total_symbols, schedule="static", nogil=True):
+        index = bit_array_to_number(bits, end=(s_i+1)*bits_per_symbol, start=s_i*bits_per_symbol)
+        a = carrier_amplitude
+        f = carrier_frequency
+        phi = carrier_phase
 
-    cdef long long loop_end = total_samples-pause
-    for i in prange(0, loop_end, nogil=True, schedule="static"):
-        index = <long long>(i/samples_per_bit)
-        a = a0 if bit_array[index] == 0 else a1
+        if is_ask:
+            a = parameters[index]
+            if a == 0:
+                continue
+        elif is_fsk:
+            f = parameters[index]
+        elif is_psk:
+            phi = parameters[index]
 
-        if a > 0:
+        for i in range(s_i * samples_per_symbol, (s_i+1)*samples_per_symbol):
             t = (i+start) / sample_rate
+            if is_gfsk:
+                f = gauss_filtered_freqs_phases[i, 0]
+                phi = gauss_filtered_freqs_phases[i, 1]
             arg = 2 * M_PI * f * t + phi
+
             result_view[i, 0] = <iq>(a * cosf(arg))
             result_view[i, 1] = <iq>(a * sinf(arg))
 
     return result
 
-cpdef modulate_psk(unsigned char[:] bit_array, unsigned long pause, unsigned long start,
-                  double a, double f, double phi0, double phi1, double sample_rate,
-                  unsigned long samples_per_bit, iq iq_type):
-    cdef long long i = 0, index = 0
-    cdef float t = 0, phi = 0, arg = 0
-    cdef long long total_samples = int(len(bit_array) * samples_per_bit + pause)
-
-    result = np.zeros((total_samples, 2), dtype=get_numpy_dtype(iq_type))
-    cdef iq[:, ::1] result_view = result
-
-    cdef long long loop_end = total_samples-pause
-    for i in prange(0, loop_end, nogil=True, schedule="static"):
-        index = <long long>(i/samples_per_bit)
-        phi = phi0 if bit_array[index] == 0 else phi1
 
-        t = (i+start) / sample_rate
-        arg = 2 * M_PI * f * t + phi
-        result_view[i, 0] = <iq>(a * cosf(arg))
-        result_view[i, 1] = <iq>(a * sinf(arg))
-
-    return result
+cdef np.ndarray[np.float32_t, ndim=2] get_gauss_filtered_freqs_phases(uint8_t[:] bits,  float[:] parameters,
+                                                                     uint32_t num_symbols, uint32_t samples_per_symbol,
+                                                                     float sample_rate, float phi, uint32_t start,
+                                                                     float gauss_bt, float filter_width):
+    cdef int64_t i, s_i, index, num_values = num_symbols * samples_per_symbol
+    cdef np.ndarray[np.float32_t, ndim=1] frequencies = np.empty(num_values, dtype=np.float32)
+    cdef uint16_t bits_per_symbol = int(len(bits) // num_symbols)
 
+    for s_i in range(0, num_symbols):
+        index = bit_array_to_number(bits, end=(s_i+1)*bits_per_symbol, start=s_i*bits_per_symbol)
 
-cdef np.ndarray[np.float32_t, ndim=1] gauss_fir(float sample_rate, unsigned long samples_per_bit,
-                                                float bt=.5, float filter_width=1.0):
-    """
+        for i in range(s_i * samples_per_symbol, (s_i+1)*samples_per_symbol):
+            frequencies[i] = parameters[index]
 
-    :param filter_width: Filter width
-    :param bt: normalized 3-dB bandwidth-symbol time product
-    :return:
-    """
-    # http://onlinelibrary.wiley.com/doi/10.1002/9780470041956.app2/pdf
-    cdef np.ndarray[np.float32_t] k = np.arange(-int(filter_width * samples_per_bit),
-                                                int(filter_width * samples_per_bit) + 1,
-                                                dtype=np.float32)
-    cdef float ts = samples_per_bit / sample_rate  # symbol time
-    cdef np.ndarray[np.float32_t] h = np.sqrt((2 * np.pi) / (np.log(2))) * bt / ts * np.exp(
-        -(((np.sqrt(2) * np.pi) / np.sqrt(np.log(2)) * bt * k / samples_per_bit) ** 2))
-    return h / h.sum()
-
-cpdef modulate_gfsk(unsigned char[:] bit_array, unsigned long pause, unsigned long start,
-                    double a, double freq0, double freq1, double phi, double sample_rate,
-                    unsigned long samples_per_bit, double gauss_bt, double filter_width, iq iq_type):
-    cdef long long i = 0, index = 0
-    cdef long long total_samples = int(len(bit_array) * samples_per_bit + pause)
-
-    cdef np.ndarray[np.float32_t, ndim=1] frequencies = np.empty(total_samples - pause, dtype=np.float32)
-    cdef long long loop_end = total_samples-pause
-
-    for i in prange(0, loop_end, nogil=True, schedule="static"):
-        index = <long long>(i/samples_per_bit)
-        frequencies[i] = freq0 if bit_array[index] == 0 else freq1
-
-    cdef np.ndarray[np.float32_t, ndim=1] t = np.arange(start, start + total_samples - pause, dtype=np.float32) / sample_rate
-    cdef np.ndarray[np.float32_t, ndim=1] gfir = gauss_fir(sample_rate, samples_per_bit,
+    cdef np.ndarray[np.float32_t, ndim=1] t = np.arange(start, start + num_values, dtype=np.float32) / sample_rate
+    cdef np.ndarray[np.float32_t, ndim=1] gfir = gauss_fir(sample_rate, samples_per_symbol,
                                                            bt=gauss_bt, filter_width=filter_width)
 
     if len(frequencies) >= len(gfir):
@@ -179,23 +126,30 @@ cpdef modulate_gfsk(unsigned char[:] bit_array, unsigned long pause, unsigned lo
         # Prevent dimension crash later, because gaussian finite impulse response is longer then param_vector
         frequencies = np.convolve(gfir, frequencies, mode="same")[:len(frequencies)]
 
-    result = np.zeros((total_samples, 2), dtype=get_numpy_dtype(iq_type))
-    cdef iq[:, ::1] result_view = result
-
-    cdef np.ndarray[np.float32_t, ndim=1] phases = np.empty(len(frequencies), dtype=np.float32)
+    cdef np.ndarray[np.float32_t, ndim=1] phases = np.zeros(len(frequencies), dtype=np.float32)
     phases[0] = phi
     for i in range(0, len(phases) - 1):
          # Correct the phase to prevent spiky jumps
         phases[i + 1] = 2 * M_PI * t[i] * (frequencies[i] - frequencies[i + 1]) + phases[i]
 
-    cdef np.ndarray[np.float32_t, ndim=1] arg = (2 * M_PI * frequencies * t + phases)
+    return np.column_stack((frequencies, phases))
 
-    cdef long long stop = max(0, total_samples-pause)
-    for i in prange(0, stop, nogil=True, schedule="static"):
-        result_view[i, 0] = <iq>(a * cosf(arg[i]))
-        result_view[i, 1] = <iq>(a * sinf(arg[i]))
+cdef np.ndarray[np.float32_t, ndim=1] gauss_fir(float sample_rate, uint32_t samples_per_symbol,
+                                                float bt=.5, float filter_width=1.0):
+    """
 
-    return result
+    :param filter_width: Filter width
+    :param bt: normalized 3-dB bandwidth-symbol time product
+    :return:
+    """
+    # http://onlinelibrary.wiley.com/doi/10.1002/9780470041956.app2/pdf
+    cdef np.ndarray[np.float32_t] k = np.arange(-int(filter_width * samples_per_symbol),
+                                                int(filter_width * samples_per_symbol) + 1,
+                                                dtype=np.float32)
+    cdef float ts = samples_per_symbol / sample_rate  # symbol time
+    cdef np.ndarray[np.float32_t] h = np.sqrt((2 * np.pi) / (np.log(2))) * bt / ts * np.exp(
+        -(((np.sqrt(2) * np.pi) / np.sqrt(np.log(2)) * bt * k / samples_per_symbol) ** 2))
+    return h / h.sum()
 
 cdef float calc_costa_alpha(float bw, float damp=1 / sqrt(2)) nogil:
     # BW in range((2pi/200), (2pi/100))

src/urh/cythonext/util.pxd

@@ -5,4 +5,8 @@ ctypedef fused iq:
     unsigned short
     float
 
-ctypedef iq[:, ::1] IQ
+ctypedef iq[:, ::1] IQ
+
+from libc.stdint cimport uint64_t, uint8_t, int64_t
+
+cpdef uint64_t bit_array_to_number(uint8_t[:] bits, int64_t end, int64_t start=*) nogil