Enhance CSV Import (#357)

* add csv import dialog

* make filename in dialog configurable

* prevent modulation bootstrap to speed up unittest

* add unittests for csv import dialog

* show dialog if show is set

* integrate csv import dialog

src/urh/controller/CSVImportDialogController.py

@@ -0,0 +1,234 @@
+import csv
+
+import os
+import numpy as np
+from PyQt5.QtCore import Qt, pyqtSlot, pyqtSignal
+from PyQt5.QtWidgets import QDialog, QInputDialog, QApplication, QCompleter, QDirModel, QFileDialog
+
+from urh.ui.ui_csv_wizard import Ui_DialogCSVImport
+from urh.util import FileOperator, util
+from urh.util.Errors import Errors
+
+
+class CSVImportDialogController(QDialog):
+    data_imported = pyqtSignal(str, float)  # Complex Filename + Sample Rate
+
+
+    PREVIEW_ROWS = 100
+    COLUMNS = {"T": 0, "I": 1, "Q": 2}
+
+    def __init__(self, filename="", parent=None):
+        super().__init__(parent)
+        self.ui = Ui_DialogCSVImport()
+        self.ui.setupUi(self)
+        self.setAttribute(Qt.WA_DeleteOnClose)
+        self.ui.btnAutoDefault.hide()
+
+        completer = QCompleter()
+        completer.setModel(QDirModel(completer))
+        self.ui.lineEditFilename.setCompleter(completer)
+
+        self.filename = None  # type: str
+        self.ui.lineEditFilename.setText(filename)
+        self.update_file()
+
+        self.ui.tableWidgetPreview.setColumnHidden(self.COLUMNS["T"], True)
+        self.update_preview()
+
+        self.create_connects()
+
+    def create_connects(self):
+        self.accepted.connect(self.on_accepted)
+        self.ui.lineEditFilename.editingFinished.connect(self.on_line_edit_filename_editing_finished)
+        self.ui.btnChooseFile.clicked.connect(self.on_btn_choose_file_clicked)
+        self.ui.btnAddSeparator.clicked.connect(self.on_btn_add_separator_clicked)
+        self.ui.comboBoxCSVSeparator.currentIndexChanged.connect(self.on_combobox_csv_separator_current_index_changed)
+        self.ui.spinBoxIDataColumn.valueChanged.connect(self.on_spinbox_i_data_column_value_changed)
+        self.ui.spinBoxQDataColumn.valueChanged.connect(self.on_spinbox_q_data_column_value_changed)
+        self.ui.spinBoxTimestampColumn.valueChanged.connect(self.on_spinbox_timestamp_value_changed)
+
+    def update_file(self):
+        filename = self.ui.lineEditFilename.text()
+        self.filename = filename
+
+        enable = util.file_can_be_opened(filename)
+        if enable:
+            with open(self.filename, encoding="utf-8-sig") as f:
+                lines = []
+                for i, line in enumerate(f):
+                    if i >= self.PREVIEW_ROWS:
+                        break
+                    lines.append(line.strip())
+                self.ui.plainTextEditFilePreview.setPlainText("\n".join(lines))
+        else:
+            self.ui.plainTextEditFilePreview.clear()
+
+        self.ui.plainTextEditFilePreview.setEnabled(enable)
+        self.ui.comboBoxCSVSeparator.setEnabled(enable)
+        self.ui.spinBoxIDataColumn.setEnabled(enable)
+        self.ui.spinBoxQDataColumn.setEnabled(enable)
+        self.ui.spinBoxTimestampColumn.setEnabled(enable)
+        self.ui.tableWidgetPreview.setEnabled(enable)
+        self.ui.labelFileNotFound.setVisible(not enable)
+
+    def update_preview(self):
+        if not util.file_can_be_opened(self.filename):
+            self.update_file()
+            return
+
+        i_data_col = self.ui.spinBoxIDataColumn.value() - 1
+        q_data_col = self.ui.spinBoxQDataColumn.value() - 1
+        timestamp_col = self.ui.spinBoxTimestampColumn.value() - 1
+
+        self.ui.tableWidgetPreview.setRowCount(self.PREVIEW_ROWS)
+
+        with open(self.filename, encoding="utf-8-sig") as f:
+            csv_reader = csv.reader(f, delimiter=self.ui.comboBoxCSVSeparator.currentText())
+            row = -1
+
+            for line in csv_reader:
+                row += 1
+                result = self.parse_csv_line(line, i_data_col, q_data_col, timestamp_col)
+                if result is not None:
+                    for key, value in result.items():
+                        self.ui.tableWidgetPreview.setItem(row, self.COLUMNS[key], util.create_table_item(value))
+                else:
+                    for col in self.COLUMNS.values():
+                        self.ui.tableWidgetPreview.setItem(row, col, util.create_table_item("Invalid"))
+
+                if row >= self.PREVIEW_ROWS - 1:
+                    break
+
+            self.ui.tableWidgetPreview.setRowCount(row + 1)
+
+    @staticmethod
+    def parse_csv_line(csv_line: str, i_data_col: int, q_data_col: int, timestamp_col: int):
+        result = dict()
+
+        if i_data_col >= 0:
+            try:
+                result["I"] = float(csv_line[i_data_col])
+            except:
+                return None
+        else:
+            result["I"] = 0.0
+
+        if q_data_col >= 0:
+            try:
+                result["Q"] = float(csv_line[q_data_col])
+            except:
+                return None
+        else:
+            result["Q"] = 0.0
+
+        if timestamp_col >= 0:
+            try:
+                result["T"] = float(csv_line[timestamp_col])
+            except:
+                return None
+
+        return result
+
+    @staticmethod
+    def parse_csv_file(filename: str, separator: str, i_data_col: int, q_data_col=-1, t_data_col=-1):
+        iq_data = []
+        timestamps = [] if t_data_col > -1 else None
+        with open(filename, encoding="utf-8-sig") as f:
+            csv_reader = csv.reader(f, delimiter=separator)
+            for line in csv_reader:
+                parsed = CSVImportDialogController.parse_csv_line(line, i_data_col, q_data_col, t_data_col)
+                if parsed is None:
+                    continue
+
+                iq_data.append(complex(parsed["I"], parsed["Q"]))
+                if timestamps is not None:
+                    timestamps.append(parsed["T"])
+
+        iq_data = np.asarray(iq_data, dtype=np.complex64)
+        sample_rate = CSVImportDialogController.estimate_sample_rate(timestamps)
+        return iq_data / abs(iq_data.max()), sample_rate
+
+    @staticmethod
+    def estimate_sample_rate(timestamps):
+        if timestamps is None or len(timestamps) < 2:
+            return None
+
+        previous_timestamp = timestamps[0]
+        durations = []
+
+        for timestamp in timestamps[1:CSVImportDialogController.PREVIEW_ROWS]:
+            durations.append(abs(timestamp-previous_timestamp))
+            previous_timestamp = timestamp
+
+        return 1 / (sum(durations) / len(durations))
+
+    @pyqtSlot()
+    def on_line_edit_filename_editing_finished(self):
+        self.update_file()
+        self.update_preview()
+
+    @pyqtSlot()
+    def on_btn_choose_file_clicked(self):
+        filename, _ = QFileDialog.getOpenFileName(self, self.tr("Choose file"), directory=FileOperator.RECENT_PATH,
+                                                  filter="CSV files (*.csv);;All files (*.*)")
+
+        if filename:
+            self.ui.lineEditFilename.setText(filename)
+            self.ui.lineEditFilename.editingFinished.emit()
+
+    @pyqtSlot()
+    def on_btn_add_separator_clicked(self):
+        sep, ok = QInputDialog.getText(self, "Enter Separator", "Separator:", text=",")
+        if ok and sep not in (self.ui.comboBoxCSVSeparator.itemText(i) for i in
+                              range(self.ui.comboBoxCSVSeparator.count())):
+            if len(sep) == 1:
+                self.ui.comboBoxCSVSeparator.addItem(sep)
+            else:
+                Errors.generic_error("Invalid Separator", "Separator must be exactly one character.")
+
+    @pyqtSlot(int)
+    def on_combobox_csv_separator_current_index_changed(self, index: int):
+        self.update_preview()
+
+    @pyqtSlot(int)
+    def on_spinbox_i_data_column_value_changed(self, value: int):
+        self.update_preview()
+
+    @pyqtSlot(int)
+    def on_spinbox_q_data_column_value_changed(self, value: int):
+        self.update_preview()
+
+    @pyqtSlot(int)
+    def on_spinbox_timestamp_value_changed(self, value: int):
+        self.ui.tableWidgetPreview.setColumnHidden(self.COLUMNS["T"], value == 0)
+        self.update_preview()
+
+    @pyqtSlot()
+    def on_accepted(self):
+        QApplication.setOverrideCursor(Qt.WaitCursor)
+
+        iq_data, sample_rate = self.parse_csv_file(self.filename, self.ui.comboBoxCSVSeparator.currentText(),
+                                                   self.ui.spinBoxIDataColumn.value()-1,
+                                                   self.ui.spinBoxQDataColumn.value()-1,
+                                                   self.ui.spinBoxTimestampColumn.value()-1)
+
+        target_filename = self.filename.rstrip(".csv")
+        if os.path.exists(target_filename + ".complex"):
+            i = 1
+            while os.path.exists(target_filename + "_" + str(i) + ".complex"):
+                i += 1
+        else:
+            i = None
+
+        target_filename = target_filename if not i else target_filename + "_" + str(i)
+        target_filename += ".complex"
+
+        iq_data.tofile(target_filename)
+
+        self.data_imported.emit(target_filename, sample_rate if sample_rate is not None else 0)
+        QApplication.restoreOverrideCursor()
+
+if __name__ == '__main__':
+    app = QApplication(["urh"])
+    csv_dia = CSVImportDialogController()
+    csv_dia.exec_()

src/urh/controller/MainController.py

@@ -8,6 +8,7 @@
     QMessageBox, QApplication, QCheckBox
 
 from urh import constants, version
+from urh.controller.CSVImportDialogController import CSVImportDialogController
 from urh.controller.CompareFrameController import CompareFrameController
 from urh.controller.DecoderWidgetController import DecoderWidgetController
 from urh.controller.GeneratorTabController import GeneratorTabController
@@ -337,7 +338,7 @@ def close_signal_frame(self, signal_frame: SignalFrameController):
             Errors.generic_error(self.tr("Failed to close"), str(e), traceback.format_exc())
             self.unsetCursor()
 
-    def add_files(self, filepaths, group_id=0):
+    def add_files(self, filepaths, group_id=0, enforce_sample_rate=None):
         num_files = len(filepaths)
         if num_files == 0:
             return
@@ -357,13 +358,13 @@ def add_files(self, filepaths, group_id=0):
             FileOperator.RECENT_PATH = os.path.split(file)[0]
 
             if file.endswith(".complex"):
-                self.add_signalfile(file, group_id)
+                self.add_signalfile(file, group_id, enforce_sample_rate=enforce_sample_rate)
             elif file.endswith(".coco"):
-                self.add_signalfile(file, group_id)
+                self.add_signalfile(file, group_id, enforce_sample_rate=enforce_sample_rate)
             elif file.endswith(".proto") or file.endswith(".proto.xml"):
                 self.add_protocol_file(file)
             elif file.endswith(".wav"):
-                self.add_signalfile(file, group_id)
+                self.add_signalfile(file, group_id, enforce_sample_rate=enforce_sample_rate)
             elif file.endswith(".fuzz") or file.endswith(".fuzz.xml"):
                 self.add_fuzz_profile(file)
             elif file.endswith(".txt"):
@@ -374,7 +375,7 @@ def add_files(self, filepaths, group_id=0):
             elif os.path.basename(file) == constants.PROJECT_FILE:
                 self.project_manager.set_project_folder(os.path.split(file)[0])
             else:
-                self.add_signalfile(file, group_id)
+                self.add_signalfile(file, group_id, enforce_sample_rate=enforce_sample_rate)
 
             if self.project_manager.project_file is None:
                 self.adjust_for_current_file(file)
@@ -831,25 +832,13 @@ def on_cancel_triggered(self):
 
     @pyqtSlot()
     def on_import_samples_from_csv_action_triggered(self):
-        dialog = QFileDialog(self)
-        dialog.setDirectory(FileOperator.RECENT_PATH)
-        dialog.setWindowTitle("Import csv")
-        dialog.setFileMode(QFileDialog.ExistingFiles)
-        dialog.setNameFilter("CSV files (*.csv);;All files (*)")
-        dialog.setOptions(QFileDialog.DontResolveSymlinks)
-        dialog.setViewMode(QFileDialog.Detail)
-
-        if dialog.exec_():
-            self.setCursor(Qt.WaitCursor)
-            for file_name in dialog.selectedFiles():
-                try:
-                    self.__import_csv(file_name)
-                except Exception as e:
-                    logger.error("Error reading csv {0}: {1}".format(file_name, e))
-            self.unsetCursor()
+        self.__import_csv(file_name="")
 
     def __import_csv(self, file_name, group_id=0):
-        self.setCursor(Qt.WaitCursor)
-        complex_file = Signal.csv_to_complex_file(file_name)
-        self.add_files([complex_file], group_id=group_id)
-        self.unsetCursor()
+        def on_data_imported(complex_file, sample_rate):
+            sample_rate = None if sample_rate == 0 else sample_rate
+            self.add_files([complex_file], group_id=group_id, enforce_sample_rate=sample_rate)
+
+        dialog = CSVImportDialogController(file_name, parent=self)
+        dialog.data_imported.connect(on_data_imported)
+        dialog.exec_()

src/urh/signalprocessing/Signal.py

@@ -460,31 +460,3 @@ def from_samples(samples: np.ndarray, name: str, sample_rate: float):
         signal._fulldata = samples
 
         return signal
-
-    @staticmethod
-    def csv_to_complex_file(csv_filename: str) -> str:
-        comments = {";", " "}
-        with open(csv_filename, encoding="utf-8-sig") as f:
-            csv_reader = csv.reader(f, delimiter=",")
-            csv_data = [line for line in csv_reader if line[0][0] not in comments]
-
-        arr = np.asarray(csv_data, dtype=np.float32)
-
-        data = np.empty(len(arr), dtype=np.complex64)
-        data.real = arr[:, 0]
-        data.imag = arr[:, 1]
-        data = data / abs(data.max())
-
-        target_filename = csv_filename.rstrip(".csv")
-        if os.path.exists(target_filename + ".complex"):
-            i = 1
-            while os.path.exists(target_filename + "_" + str(i) + ".complex"):
-                i += 1
-        else:
-            i = None
-
-        target_filename = target_filename if not i else target_filename + "_" + str(i)
-        target_filename += ".complex"
-
-        data.tofile(target_filename)
-        return target_filename