From 37c4cc2bc0950d30da65a3004b5c35b7477a9223 Mon Sep 17 00:00:00 2001 From: Johannes Pohl Date: Wed, 15 Nov 2017 16:28:05 +0100 Subject: [PATCH] Enhance CSV Import (#357) * add csv import dialog * make filename in dialog configurable * prevent modulation bootstrap to speed up unittest * add unittests for csv import dialog * show dialog if show is set * integrate csv import dialog --- .../controller/CSVImportDialogController.py | 234 +++++++++++++++ src/urh/controller/MainController.py | 39 +-- src/urh/signalprocessing/Signal.py | 28 -- src/urh/ui/ui_csv_wizard.py | 151 ++++++++++ src/urh/util/util.py | 19 +- tests/test_csv_import_dialog.py | 89 ++++++ tests/test_fuzzing_dialog.py | 3 + tests/test_maincontroller_gui.py | 17 +- ui/csv_wizard.ui | 280 ++++++++++++++++++ 9 files changed, 805 insertions(+), 55 deletions(-) create mode 100644 src/urh/controller/CSVImportDialogController.py create mode 100644 src/urh/ui/ui_csv_wizard.py create mode 100644 tests/test_csv_import_dialog.py create mode 100644 ui/csv_wizard.ui diff --git a/src/urh/controller/CSVImportDialogController.py b/src/urh/controller/CSVImportDialogController.py new file mode 100644 index 0000000000..95bafdbf0b --- /dev/null +++ b/src/urh/controller/CSVImportDialogController.py @@ -0,0 +1,234 @@ +import csv + +import os +import numpy as np +from PyQt5.QtCore import Qt, pyqtSlot, pyqtSignal +from PyQt5.QtWidgets import QDialog, QInputDialog, QApplication, QCompleter, QDirModel, QFileDialog + +from urh.ui.ui_csv_wizard import Ui_DialogCSVImport +from urh.util import FileOperator, util +from urh.util.Errors import Errors + + +class CSVImportDialogController(QDialog): + data_imported = pyqtSignal(str, float) # Complex Filename + Sample Rate + + + PREVIEW_ROWS = 100 + COLUMNS = {"T": 0, "I": 1, "Q": 2} + + def __init__(self, filename="", parent=None): + super().__init__(parent) + self.ui = Ui_DialogCSVImport() + self.ui.setupUi(self) + self.setAttribute(Qt.WA_DeleteOnClose) + self.ui.btnAutoDefault.hide() + + completer = QCompleter() + completer.setModel(QDirModel(completer)) + self.ui.lineEditFilename.setCompleter(completer) + + self.filename = None # type: str + self.ui.lineEditFilename.setText(filename) + self.update_file() + + self.ui.tableWidgetPreview.setColumnHidden(self.COLUMNS["T"], True) + self.update_preview() + + self.create_connects() + + def create_connects(self): + self.accepted.connect(self.on_accepted) + self.ui.lineEditFilename.editingFinished.connect(self.on_line_edit_filename_editing_finished) + self.ui.btnChooseFile.clicked.connect(self.on_btn_choose_file_clicked) + self.ui.btnAddSeparator.clicked.connect(self.on_btn_add_separator_clicked) + self.ui.comboBoxCSVSeparator.currentIndexChanged.connect(self.on_combobox_csv_separator_current_index_changed) + self.ui.spinBoxIDataColumn.valueChanged.connect(self.on_spinbox_i_data_column_value_changed) + self.ui.spinBoxQDataColumn.valueChanged.connect(self.on_spinbox_q_data_column_value_changed) + self.ui.spinBoxTimestampColumn.valueChanged.connect(self.on_spinbox_timestamp_value_changed) + + def update_file(self): + filename = self.ui.lineEditFilename.text() + self.filename = filename + + enable = util.file_can_be_opened(filename) + if enable: + with open(self.filename, encoding="utf-8-sig") as f: + lines = [] + for i, line in enumerate(f): + if i >= self.PREVIEW_ROWS: + break + lines.append(line.strip()) + self.ui.plainTextEditFilePreview.setPlainText("\n".join(lines)) + else: + self.ui.plainTextEditFilePreview.clear() + + self.ui.plainTextEditFilePreview.setEnabled(enable) + self.ui.comboBoxCSVSeparator.setEnabled(enable) + self.ui.spinBoxIDataColumn.setEnabled(enable) + self.ui.spinBoxQDataColumn.setEnabled(enable) + self.ui.spinBoxTimestampColumn.setEnabled(enable) + self.ui.tableWidgetPreview.setEnabled(enable) + self.ui.labelFileNotFound.setVisible(not enable) + + def update_preview(self): + if not util.file_can_be_opened(self.filename): + self.update_file() + return + + i_data_col = self.ui.spinBoxIDataColumn.value() - 1 + q_data_col = self.ui.spinBoxQDataColumn.value() - 1 + timestamp_col = self.ui.spinBoxTimestampColumn.value() - 1 + + self.ui.tableWidgetPreview.setRowCount(self.PREVIEW_ROWS) + + with open(self.filename, encoding="utf-8-sig") as f: + csv_reader = csv.reader(f, delimiter=self.ui.comboBoxCSVSeparator.currentText()) + row = -1 + + for line in csv_reader: + row += 1 + result = self.parse_csv_line(line, i_data_col, q_data_col, timestamp_col) + if result is not None: + for key, value in result.items(): + self.ui.tableWidgetPreview.setItem(row, self.COLUMNS[key], util.create_table_item(value)) + else: + for col in self.COLUMNS.values(): + self.ui.tableWidgetPreview.setItem(row, col, util.create_table_item("Invalid")) + + if row >= self.PREVIEW_ROWS - 1: + break + + self.ui.tableWidgetPreview.setRowCount(row + 1) + + @staticmethod + def parse_csv_line(csv_line: str, i_data_col: int, q_data_col: int, timestamp_col: int): + result = dict() + + if i_data_col >= 0: + try: + result["I"] = float(csv_line[i_data_col]) + except: + return None + else: + result["I"] = 0.0 + + if q_data_col >= 0: + try: + result["Q"] = float(csv_line[q_data_col]) + except: + return None + else: + result["Q"] = 0.0 + + if timestamp_col >= 0: + try: + result["T"] = float(csv_line[timestamp_col]) + except: + return None + + return result + + @staticmethod + def parse_csv_file(filename: str, separator: str, i_data_col: int, q_data_col=-1, t_data_col=-1): + iq_data = [] + timestamps = [] if t_data_col > -1 else None + with open(filename, encoding="utf-8-sig") as f: + csv_reader = csv.reader(f, delimiter=separator) + for line in csv_reader: + parsed = CSVImportDialogController.parse_csv_line(line, i_data_col, q_data_col, t_data_col) + if parsed is None: + continue + + iq_data.append(complex(parsed["I"], parsed["Q"])) + if timestamps is not None: + timestamps.append(parsed["T"]) + + iq_data = np.asarray(iq_data, dtype=np.complex64) + sample_rate = CSVImportDialogController.estimate_sample_rate(timestamps) + return iq_data / abs(iq_data.max()), sample_rate + + @staticmethod + def estimate_sample_rate(timestamps): + if timestamps is None or len(timestamps) < 2: + return None + + previous_timestamp = timestamps[0] + durations = [] + + for timestamp in timestamps[1:CSVImportDialogController.PREVIEW_ROWS]: + durations.append(abs(timestamp-previous_timestamp)) + previous_timestamp = timestamp + + return 1 / (sum(durations) / len(durations)) + + @pyqtSlot() + def on_line_edit_filename_editing_finished(self): + self.update_file() + self.update_preview() + + @pyqtSlot() + def on_btn_choose_file_clicked(self): + filename, _ = QFileDialog.getOpenFileName(self, self.tr("Choose file"), directory=FileOperator.RECENT_PATH, + filter="CSV files (*.csv);;All files (*.*)") + + if filename: + self.ui.lineEditFilename.setText(filename) + self.ui.lineEditFilename.editingFinished.emit() + + @pyqtSlot() + def on_btn_add_separator_clicked(self): + sep, ok = QInputDialog.getText(self, "Enter Separator", "Separator:", text=",") + if ok and sep not in (self.ui.comboBoxCSVSeparator.itemText(i) for i in + range(self.ui.comboBoxCSVSeparator.count())): + if len(sep) == 1: + self.ui.comboBoxCSVSeparator.addItem(sep) + else: + Errors.generic_error("Invalid Separator", "Separator must be exactly one character.") + + @pyqtSlot(int) + def on_combobox_csv_separator_current_index_changed(self, index: int): + self.update_preview() + + @pyqtSlot(int) + def on_spinbox_i_data_column_value_changed(self, value: int): + self.update_preview() + + @pyqtSlot(int) + def on_spinbox_q_data_column_value_changed(self, value: int): + self.update_preview() + + @pyqtSlot(int) + def on_spinbox_timestamp_value_changed(self, value: int): + self.ui.tableWidgetPreview.setColumnHidden(self.COLUMNS["T"], value == 0) + self.update_preview() + + @pyqtSlot() + def on_accepted(self): + QApplication.setOverrideCursor(Qt.WaitCursor) + + iq_data, sample_rate = self.parse_csv_file(self.filename, self.ui.comboBoxCSVSeparator.currentText(), + self.ui.spinBoxIDataColumn.value()-1, + self.ui.spinBoxQDataColumn.value()-1, + self.ui.spinBoxTimestampColumn.value()-1) + + target_filename = self.filename.rstrip(".csv") + if os.path.exists(target_filename + ".complex"): + i = 1 + while os.path.exists(target_filename + "_" + str(i) + ".complex"): + i += 1 + else: + i = None + + target_filename = target_filename if not i else target_filename + "_" + str(i) + target_filename += ".complex" + + iq_data.tofile(target_filename) + + self.data_imported.emit(target_filename, sample_rate if sample_rate is not None else 0) + QApplication.restoreOverrideCursor() + +if __name__ == '__main__': + app = QApplication(["urh"]) + csv_dia = CSVImportDialogController() + csv_dia.exec_() diff --git a/src/urh/controller/MainController.py b/src/urh/controller/MainController.py index e6ae28ef2a..b02e156010 100644 --- a/src/urh/controller/MainController.py +++ b/src/urh/controller/MainController.py @@ -8,6 +8,7 @@ QMessageBox, QApplication, QCheckBox from urh import constants, version +from urh.controller.CSVImportDialogController import CSVImportDialogController from urh.controller.CompareFrameController import CompareFrameController from urh.controller.DecoderWidgetController import DecoderWidgetController from urh.controller.GeneratorTabController import GeneratorTabController @@ -337,7 +338,7 @@ def close_signal_frame(self, signal_frame: SignalFrameController): Errors.generic_error(self.tr("Failed to close"), str(e), traceback.format_exc()) self.unsetCursor() - def add_files(self, filepaths, group_id=0): + def add_files(self, filepaths, group_id=0, enforce_sample_rate=None): num_files = len(filepaths) if num_files == 0: return @@ -357,13 +358,13 @@ def add_files(self, filepaths, group_id=0): FileOperator.RECENT_PATH = os.path.split(file)[0] if file.endswith(".complex"): - self.add_signalfile(file, group_id) + self.add_signalfile(file, group_id, enforce_sample_rate=enforce_sample_rate) elif file.endswith(".coco"): - self.add_signalfile(file, group_id) + self.add_signalfile(file, group_id, enforce_sample_rate=enforce_sample_rate) elif file.endswith(".proto") or file.endswith(".proto.xml"): self.add_protocol_file(file) elif file.endswith(".wav"): - self.add_signalfile(file, group_id) + self.add_signalfile(file, group_id, enforce_sample_rate=enforce_sample_rate) elif file.endswith(".fuzz") or file.endswith(".fuzz.xml"): self.add_fuzz_profile(file) elif file.endswith(".txt"): @@ -374,7 +375,7 @@ def add_files(self, filepaths, group_id=0): elif os.path.basename(file) == constants.PROJECT_FILE: self.project_manager.set_project_folder(os.path.split(file)[0]) else: - self.add_signalfile(file, group_id) + self.add_signalfile(file, group_id, enforce_sample_rate=enforce_sample_rate) if self.project_manager.project_file is None: self.adjust_for_current_file(file) @@ -831,25 +832,13 @@ def on_cancel_triggered(self): @pyqtSlot() def on_import_samples_from_csv_action_triggered(self): - dialog = QFileDialog(self) - dialog.setDirectory(FileOperator.RECENT_PATH) - dialog.setWindowTitle("Import csv") - dialog.setFileMode(QFileDialog.ExistingFiles) - dialog.setNameFilter("CSV files (*.csv);;All files (*)") - dialog.setOptions(QFileDialog.DontResolveSymlinks) - dialog.setViewMode(QFileDialog.Detail) - - if dialog.exec_(): - self.setCursor(Qt.WaitCursor) - for file_name in dialog.selectedFiles(): - try: - self.__import_csv(file_name) - except Exception as e: - logger.error("Error reading csv {0}: {1}".format(file_name, e)) - self.unsetCursor() + self.__import_csv(file_name="") def __import_csv(self, file_name, group_id=0): - self.setCursor(Qt.WaitCursor) - complex_file = Signal.csv_to_complex_file(file_name) - self.add_files([complex_file], group_id=group_id) - self.unsetCursor() + def on_data_imported(complex_file, sample_rate): + sample_rate = None if sample_rate == 0 else sample_rate + self.add_files([complex_file], group_id=group_id, enforce_sample_rate=sample_rate) + + dialog = CSVImportDialogController(file_name, parent=self) + dialog.data_imported.connect(on_data_imported) + dialog.exec_() diff --git a/src/urh/signalprocessing/Signal.py b/src/urh/signalprocessing/Signal.py index 5ec3729d35..9af1b97769 100644 --- a/src/urh/signalprocessing/Signal.py +++ b/src/urh/signalprocessing/Signal.py @@ -460,31 +460,3 @@ def from_samples(samples: np.ndarray, name: str, sample_rate: float): signal._fulldata = samples return signal - - @staticmethod - def csv_to_complex_file(csv_filename: str) -> str: - comments = {";", " "} - with open(csv_filename, encoding="utf-8-sig") as f: - csv_reader = csv.reader(f, delimiter=",") - csv_data = [line for line in csv_reader if line[0][0] not in comments] - - arr = np.asarray(csv_data, dtype=np.float32) - - data = np.empty(len(arr), dtype=np.complex64) - data.real = arr[:, 0] - data.imag = arr[:, 1] - data = data / abs(data.max()) - - target_filename = csv_filename.rstrip(".csv") - if os.path.exists(target_filename + ".complex"): - i = 1 - while os.path.exists(target_filename + "_" + str(i) + ".complex"): - i += 1 - else: - i = None - - target_filename = target_filename if not i else target_filename + "_" + str(i) - target_filename += ".complex" - - data.tofile(target_filename) - return target_filename diff --git a/src/urh/ui/ui_csv_wizard.py b/src/urh/ui/ui_csv_wizard.py new file mode 100644 index 0000000000..6d8db12d52 --- /dev/null +++ b/src/urh/ui/ui_csv_wizard.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- + +# +# +# WARNING! All changes made in this file will be lost! + +from PyQt5 import QtCore, QtGui, QtWidgets + +class Ui_DialogCSVImport(object): + def setupUi(self, DialogCSVImport): + DialogCSVImport.setObjectName("DialogCSVImport") + DialogCSVImport.resize(635, 674) + self.gridLayout = QtWidgets.QGridLayout(DialogCSVImport) + self.gridLayout.setObjectName("gridLayout") + self.labelFileNotFound = QtWidgets.QLabel(DialogCSVImport) + self.labelFileNotFound.setObjectName("labelFileNotFound") + self.gridLayout.addWidget(self.labelFileNotFound, 1, 2, 1, 1) + self.horizontalLayout = QtWidgets.QHBoxLayout() + self.horizontalLayout.setObjectName("horizontalLayout") + self.comboBoxCSVSeparator = QtWidgets.QComboBox(DialogCSVImport) + self.comboBoxCSVSeparator.setObjectName("comboBoxCSVSeparator") + self.comboBoxCSVSeparator.addItem("") + self.comboBoxCSVSeparator.addItem("") + self.horizontalLayout.addWidget(self.comboBoxCSVSeparator) + self.btnAddSeparator = QtWidgets.QToolButton(DialogCSVImport) + icon = QtGui.QIcon.fromTheme("list-add") + self.btnAddSeparator.setIcon(icon) + self.btnAddSeparator.setIconSize(QtCore.QSize(16, 16)) + self.btnAddSeparator.setObjectName("btnAddSeparator") + self.horizontalLayout.addWidget(self.btnAddSeparator) + self.gridLayout.addLayout(self.horizontalLayout, 3, 2, 1, 1) + self.spinBoxTimestampColumn = QtWidgets.QSpinBox(DialogCSVImport) + self.spinBoxTimestampColumn.setMaximum(999999999) + self.spinBoxTimestampColumn.setObjectName("spinBoxTimestampColumn") + self.gridLayout.addWidget(self.spinBoxTimestampColumn, 6, 2, 1, 1) + self.label = QtWidgets.QLabel(DialogCSVImport) + self.label.setObjectName("label") + self.gridLayout.addWidget(self.label, 4, 0, 1, 2) + self.spinBoxQDataColumn = QtWidgets.QSpinBox(DialogCSVImport) + self.spinBoxQDataColumn.setMaximum(999999999) + self.spinBoxQDataColumn.setObjectName("spinBoxQDataColumn") + self.gridLayout.addWidget(self.spinBoxQDataColumn, 5, 2, 1, 1) + self.label_3 = QtWidgets.QLabel(DialogCSVImport) + self.label_3.setObjectName("label_3") + self.gridLayout.addWidget(self.label_3, 6, 0, 1, 2) + self.label_2 = QtWidgets.QLabel(DialogCSVImport) + self.label_2.setObjectName("label_2") + self.gridLayout.addWidget(self.label_2, 5, 0, 1, 2) + self.groupBox = QtWidgets.QGroupBox(DialogCSVImport) + self.groupBox.setObjectName("groupBox") + self.verticalLayout_2 = QtWidgets.QVBoxLayout(self.groupBox) + self.verticalLayout_2.setObjectName("verticalLayout_2") + self.tableWidgetPreview = QtWidgets.QTableWidget(self.groupBox) + self.tableWidgetPreview.setAlternatingRowColors(True) + self.tableWidgetPreview.setObjectName("tableWidgetPreview") + self.tableWidgetPreview.setColumnCount(3) + self.tableWidgetPreview.setRowCount(0) + item = QtWidgets.QTableWidgetItem() + self.tableWidgetPreview.setHorizontalHeaderItem(0, item) + item = QtWidgets.QTableWidgetItem() + self.tableWidgetPreview.setHorizontalHeaderItem(1, item) + item = QtWidgets.QTableWidgetItem() + self.tableWidgetPreview.setHorizontalHeaderItem(2, item) + self.tableWidgetPreview.horizontalHeader().setCascadingSectionResizes(False) + self.tableWidgetPreview.horizontalHeader().setStretchLastSection(False) + self.tableWidgetPreview.verticalHeader().setStretchLastSection(False) + self.verticalLayout_2.addWidget(self.tableWidgetPreview) + self.gridLayout.addWidget(self.groupBox, 7, 0, 1, 3) + self.horizontalLayout_2 = QtWidgets.QHBoxLayout() + self.horizontalLayout_2.setObjectName("horizontalLayout_2") + self.lineEditFilename = QtWidgets.QLineEdit(DialogCSVImport) + self.lineEditFilename.setObjectName("lineEditFilename") + self.horizontalLayout_2.addWidget(self.lineEditFilename) + self.btnChooseFile = QtWidgets.QToolButton(DialogCSVImport) + self.btnChooseFile.setToolButtonStyle(QtCore.Qt.ToolButtonTextOnly) + self.btnChooseFile.setObjectName("btnChooseFile") + self.horizontalLayout_2.addWidget(self.btnChooseFile) + self.gridLayout.addLayout(self.horizontalLayout_2, 0, 2, 1, 1) + self.spinBoxIDataColumn = QtWidgets.QSpinBox(DialogCSVImport) + self.spinBoxIDataColumn.setMinimum(1) + self.spinBoxIDataColumn.setMaximum(999999999) + self.spinBoxIDataColumn.setProperty("value", 1) + self.spinBoxIDataColumn.setObjectName("spinBoxIDataColumn") + self.gridLayout.addWidget(self.spinBoxIDataColumn, 4, 2, 1, 1) + self.label_4 = QtWidgets.QLabel(DialogCSVImport) + self.label_4.setObjectName("label_4") + self.gridLayout.addWidget(self.label_4, 3, 0, 1, 2) + self.buttonBox = QtWidgets.QDialogButtonBox(DialogCSVImport) + self.buttonBox.setOrientation(QtCore.Qt.Horizontal) + self.buttonBox.setStandardButtons(QtWidgets.QDialogButtonBox.Cancel|QtWidgets.QDialogButtonBox.Ok) + self.buttonBox.setObjectName("buttonBox") + self.gridLayout.addWidget(self.buttonBox, 9, 2, 1, 1) + self.groupBoxFilePreview = QtWidgets.QGroupBox(DialogCSVImport) + self.groupBoxFilePreview.setObjectName("groupBoxFilePreview") + self.verticalLayout = QtWidgets.QVBoxLayout(self.groupBoxFilePreview) + self.verticalLayout.setObjectName("verticalLayout") + self.plainTextEditFilePreview = QtWidgets.QPlainTextEdit(self.groupBoxFilePreview) + self.plainTextEditFilePreview.setUndoRedoEnabled(False) + self.plainTextEditFilePreview.setReadOnly(True) + self.plainTextEditFilePreview.setObjectName("plainTextEditFilePreview") + self.verticalLayout.addWidget(self.plainTextEditFilePreview) + self.gridLayout.addWidget(self.groupBoxFilePreview, 2, 0, 1, 3) + self.label_5 = QtWidgets.QLabel(DialogCSVImport) + self.label_5.setObjectName("label_5") + self.gridLayout.addWidget(self.label_5, 0, 0, 1, 1) + self.btnAutoDefault = QtWidgets.QPushButton(DialogCSVImport) + self.btnAutoDefault.setDefault(True) + self.btnAutoDefault.setObjectName("btnAutoDefault") + self.gridLayout.addWidget(self.btnAutoDefault, 10, 2, 1, 1) + + self.retranslateUi(DialogCSVImport) + self.buttonBox.accepted.connect(DialogCSVImport.accept) + self.buttonBox.rejected.connect(DialogCSVImport.reject) + QtCore.QMetaObject.connectSlotsByName(DialogCSVImport) + DialogCSVImport.setTabOrder(self.lineEditFilename, self.btnChooseFile) + DialogCSVImport.setTabOrder(self.btnChooseFile, self.plainTextEditFilePreview) + DialogCSVImport.setTabOrder(self.plainTextEditFilePreview, self.comboBoxCSVSeparator) + DialogCSVImport.setTabOrder(self.comboBoxCSVSeparator, self.btnAddSeparator) + DialogCSVImport.setTabOrder(self.btnAddSeparator, self.spinBoxIDataColumn) + DialogCSVImport.setTabOrder(self.spinBoxIDataColumn, self.spinBoxQDataColumn) + DialogCSVImport.setTabOrder(self.spinBoxQDataColumn, self.spinBoxTimestampColumn) + DialogCSVImport.setTabOrder(self.spinBoxTimestampColumn, self.tableWidgetPreview) + + def retranslateUi(self, DialogCSVImport): + _translate = QtCore.QCoreApplication.translate + DialogCSVImport.setWindowTitle(_translate("DialogCSVImport", "CSV Import")) + self.labelFileNotFound.setText(_translate("DialogCSVImport", "

Could not open the selected file.

")) + self.comboBoxCSVSeparator.setItemText(0, _translate("DialogCSVImport", ",")) + self.comboBoxCSVSeparator.setItemText(1, _translate("DialogCSVImport", ";")) + self.btnAddSeparator.setToolTip(_translate("DialogCSVImport", "Add a custom separator.")) + self.btnAddSeparator.setText(_translate("DialogCSVImport", "...")) + self.spinBoxTimestampColumn.setToolTip(_translate("DialogCSVImport", "

If your dataset contains timestamps URH will calculate the sample rate from them. You can manually edit the sample rate after import in the signal details.

")) + self.spinBoxTimestampColumn.setSpecialValueText(_translate("DialogCSVImport", "Not present")) + self.label.setText(_translate("DialogCSVImport", "I Data Column:")) + self.spinBoxQDataColumn.setSpecialValueText(_translate("DialogCSVImport", "Not present")) + self.label_3.setToolTip(_translate("DialogCSVImport", "

If your dataset contains timestamps URH will calculate the sample rate from them. You can manually edit the sample rate after import in the signal details.

")) + self.label_3.setText(_translate("DialogCSVImport", "Timestamp Column:")) + self.label_2.setText(_translate("DialogCSVImport", "Q Data Column:")) + self.groupBox.setTitle(_translate("DialogCSVImport", "Preview")) + item = self.tableWidgetPreview.horizontalHeaderItem(0) + item.setText(_translate("DialogCSVImport", "Timestamp")) + item = self.tableWidgetPreview.horizontalHeaderItem(1) + item.setText(_translate("DialogCSVImport", "I")) + item = self.tableWidgetPreview.horizontalHeaderItem(2) + item.setText(_translate("DialogCSVImport", "Q")) + self.btnChooseFile.setText(_translate("DialogCSVImport", "...")) + self.label_4.setText(_translate("DialogCSVImport", "CSV Separator:")) + self.groupBoxFilePreview.setTitle(_translate("DialogCSVImport", "File Content (at most 100 rows)")) + self.label_5.setText(_translate("DialogCSVImport", "File to import:")) + self.btnAutoDefault.setText(_translate("DialogCSVImport", "Prevent Dialog From Close with Enter")) + diff --git a/src/urh/util/util.py b/src/urh/util/util.py index 34d3b17434..c27837ed4f 100644 --- a/src/urh/util/util.py +++ b/src/urh/util/util.py @@ -2,8 +2,9 @@ import os import sys +from PyQt5.QtCore import Qt from PyQt5.QtGui import QIcon -from PyQt5.QtWidgets import QDialog, QVBoxLayout, QPlainTextEdit +from PyQt5.QtWidgets import QDialog, QVBoxLayout, QPlainTextEdit, QTableWidgetItem from urh import constants from urh.util.Logger import logger @@ -119,3 +120,19 @@ def aggregate_bits(bits: array.array, size=4) -> array.array: def clip(value, minimum, maximum): return max(minimum, min(value, maximum)) + + +def file_can_be_opened(filename: str): + try: + open(filename, "r").close() + return True + except Exception as e: + if not isinstance(e, FileNotFoundError): + logger.debug(str(e)) + return False + + +def create_table_item(content): + item = QTableWidgetItem(str(content)) + item.setFlags(item.flags() & ~Qt.ItemIsEditable) + return item diff --git a/tests/test_csv_import_dialog.py b/tests/test_csv_import_dialog.py new file mode 100644 index 0000000000..c721aa47be --- /dev/null +++ b/tests/test_csv_import_dialog.py @@ -0,0 +1,89 @@ +import os +import random +import tempfile + +from tests.QtTestCase import QtTestCase +from urh.controller.CSVImportDialogController import CSVImportDialogController + + +class TestCSVImportDialog(QtTestCase): + def setUp(self): + super().setUp() + self.dialog = CSVImportDialogController() + + if self.SHOW: + self.dialog.show() + + self.i_column = self.dialog.COLUMNS["I"] + self.q_column = self.dialog.COLUMNS["Q"] + self.t_column = self.dialog.COLUMNS["T"] + + def test_invalid_file(self): + if self.SHOW: + self.assertTrue(self.dialog.ui.labelFileNotFound.isVisible()) + + self.dialog.ui.lineEditFilename.setText("/this/file/does/not/exist") + self.dialog.ui.lineEditFilename.editingFinished.emit() + self.assertEqual(self.dialog.ui.plainTextEditFilePreview.toPlainText(), "") + self.assertEqual(self.dialog.ui.tableWidgetPreview.rowCount(), 0) + + def test_comma_separated_file(self): + filename = os.path.join(tempfile.gettempdir(), "comma.csv") + with open(filename, "w") as f: + f.write("this is a comment\n") + f.write("format is\n") + f.write("Timestamp I Q Trash\n") + + for i in range(150): + f.write("{},{},{},{}\n".format(i / 1e6, i, random.uniform(0, 1), 42 * i)) + + self.dialog.ui.lineEditFilename.setText(filename) + self.dialog.ui.lineEditFilename.editingFinished.emit() + + self.dialog.ui.spinBoxIDataColumn.setValue(2) + self.dialog.ui.spinBoxTimestampColumn.setValue(1) + self.dialog.ui.spinBoxQDataColumn.setValue(3) + + for i in range(3): + for j in range(3): + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(i, j).text(), "Invalid") + + file_preview = self.dialog.ui.plainTextEditFilePreview.toPlainText() + self.assertEqual(len(file_preview.split("\n")), 100) + + self.assertEqual(self.dialog.ui.tableWidgetPreview.rowCount(), 100) + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(3, self.i_column).text(), "0.0") + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(99, self.i_column).text(), "96.0") + + last_preview_line = file_preview.split("\n")[-1] + t, i, q, _ = map(float, last_preview_line.split(",")) + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(99, self.i_column).text(), str(i)) + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(99, self.q_column).text(), str(q)) + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(99, self.t_column).text(), str(t)) + + def test_semicolon_separated_file(self): + filename = os.path.join(tempfile.gettempdir(), "semicolon.csv") + with open(filename, "w") as f: + f.write("I;Trash\n") + + for i in range(20): + f.write("{};{}\n".format(i, 24 * i)) + + self.dialog.ui.lineEditFilename.setText(filename) + self.dialog.ui.lineEditFilename.editingFinished.emit() + self.dialog.ui.comboBoxCSVSeparator.setCurrentText(";") + + self.assertTrue(self.dialog.ui.tableWidgetPreview.isColumnHidden(self.t_column)) + self.assertEqual(self.dialog.ui.tableWidgetPreview.rowCount(), 21) + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(0, self.i_column).text(), "Invalid") + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(0, self.q_column).text(), "Invalid") + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(1, self.i_column).text(), "0.0") + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(1, self.q_column).text(), "0.0") + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(2, self.i_column).text(), "1.0") + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(2, self.q_column).text(), "0.0") + + file_preview = self.dialog.ui.plainTextEditFilePreview.toPlainText() + last_preview_line = file_preview.split("\n")[-1] + i, _ = map(float, last_preview_line.split(";")) + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(20, self.i_column).text(), str(i)) + self.assertEqual(self.dialog.ui.tableWidgetPreview.item(20, self.q_column).text(), "0.0") diff --git a/tests/test_fuzzing_dialog.py b/tests/test_fuzzing_dialog.py index cce868ce31..b63ad3edd3 100644 --- a/tests/test_fuzzing_dialog.py +++ b/tests/test_fuzzing_dialog.py @@ -6,6 +6,7 @@ from urh import constants from urh.controller.FuzzingDialogController import FuzzingDialogController from urh.signalprocessing.Encoding import Encoding +from urh.signalprocessing.Modulator import Modulator class TestFuzzingDialog(QtTestCase): @@ -21,6 +22,8 @@ def setUp(self): self.gframe = self.form.generator_tab_controller self.gframe.ui.cbViewType.setCurrentIndex(1) # hex view + self.gframe.modulators.append(Modulator("Prevent Modulation bootstrap when adding first protocol")) + self.gframe.refresh_modulators() # Dewhitening mit SyncByte 0x9a7d9a7d, Data Whitening Poly 0x21, Compute and apply CRC16 via X0r, # Rest auf False anlegen und setzen diff --git a/tests/test_maincontroller_gui.py b/tests/test_maincontroller_gui.py index 4b0f07ca5a..165e84e8f2 100644 --- a/tests/test_maincontroller_gui.py +++ b/tests/test_maincontroller_gui.py @@ -1,9 +1,11 @@ import os import tempfile +from PyQt5.QtCore import QTimer from PyQt5.QtWidgets import QApplication from tests.QtTestCase import QtTestCase +from urh.controller.CSVImportDialogController import CSVImportDialogController from urh.controller.OptionsController import OptionsController @@ -61,14 +63,27 @@ def test_open_options_dialog(self): self.assertEqual(w.ui.tabWidget.currentIndex(), 1) w.close() + def __accept_csv_dialog(self): + w = next((w for w in QApplication.topLevelWidgets() if isinstance(w, CSVImportDialogController)), None) + w.accept() + def test_import_csv(self): + timer = QTimer() + timer.setInterval(10) + timer.setSingleShot(True) + timer.timeout.connect(self.__accept_csv_dialog) + self.assertEqual(self.form.signal_tab_controller.num_frames, 0) + timer.start() self.form.add_files([self.get_path_for_filename("csvtest.csv")]) + self.assertEqual(self.form.signal_tab_controller.signal_frames[0].signal.num_samples, 100) self.assertTrue(os.path.isfile(self.get_path_for_filename("csvtest.complex"))) + timer.start() self.form.add_files([self.get_path_for_filename("csvtest.csv")]) + self.assertEqual(self.form.signal_tab_controller.num_frames, 2) self.assertTrue(os.path.isfile(self.get_path_for_filename("csvtest_1.complex"))) os.remove(self.get_path_for_filename("csvtest.complex")) - os.remove(self.get_path_for_filename("csvtest_1.complex")) \ No newline at end of file + os.remove(self.get_path_for_filename("csvtest_1.complex")) diff --git a/ui/csv_wizard.ui b/ui/csv_wizard.ui new file mode 100644 index 0000000000..e0019096fe --- /dev/null +++ b/ui/csv_wizard.ui @@ -0,0 +1,280 @@ + + + DialogCSVImport + + + + 0 + 0 + 635 + 674 + + + + CSV Import + + + + + + <html><head/><body><p><span style=" color:#ff0000;">Could not open the selected file.</span></p></body></html> + + + + + + + + + + , + + + + + ; + + + + + + + + Add a custom separator. + + + ... + + + + .. + + + + 16 + 16 + + + + + + + + + + <html><head/><body><p> If your dataset contains timestamps URH will calculate the sample rate from them. You can manually edit the sample rate after import in the signal details.</p></body></html> + + + Not present + + + 999999999 + + + + + + + I Data Column: + + + + + + + Not present + + + 999999999 + + + + + + + <html><head/><body><p> If your dataset contains timestamps URH will calculate the sample rate from them. You can manually edit the sample rate after import in the signal details.</p></body></html> + + + Timestamp Column: + + + + + + + Q Data Column: + + + + + + + Preview + + + + + + true + + + false + + + false + + + false + + + + Timestamp + + + + + I + + + + + Q + + + + + + + + + + + + + + + + ... + + + Qt::ToolButtonTextOnly + + + + + + + + + 1 + + + 999999999 + + + 1 + + + + + + + CSV Separator: + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + File Content (at most 100 rows) + + + + + + false + + + true + + + + + + + + + + File to import: + + + + + + + Prevent Dialog From Close with Enter + + + true + + + + + + + lineEditFilename + btnChooseFile + plainTextEditFilePreview + comboBoxCSVSeparator + btnAddSeparator + spinBoxIDataColumn + spinBoxQDataColumn + spinBoxTimestampColumn + tableWidgetPreview + + + + + buttonBox + accepted() + DialogCSVImport + accept() + + + 375 + 641 + + + 157 + 274 + + + + + buttonBox + rejected() + DialogCSVImport + reject() + + + 443 + 647 + + + 286 + 274 + + + + +