Prebuild device extensions (#462)

* precompile device extensions

* set lib path

* prevent double build

* download libs in travis

* fix unittest

* update path

* protect access

* adapt shared path to linux

* sort os listdir

* ensure libstdc++ loaded first

* change name

* remove required

* improve test

* add usrp test

* set shared library path when processes spawn

* test pypi release with travis

* test pypi release with travis

* test pypi release with travis

* skip cleanup

* fix pypi test link

* test wheelhouse build

* skip tests

* fix path

* fix dir

* cd repo

* quote var

* do not use for loop

* add missing quote

* quote dollar

* script deployment

* fix twine upload

* deploy libs only for windows

* add 32 bit support

* fix command

* disable 32 bit

* push version for test

* push ver

* prepare merge

* trigger build

* fix windows path

* improve shortcut

* remove debug

* edit copyright

.travis.yml

@@ -82,7 +82,7 @@ script:
   - urh --version
   - urh autoclose
 
-after_success: 
+after_success:
   - |
     if [[ $TRAVIS_PYTHON_VERSION == "3.6"* ]]
     then

appveyor.yml

@@ -42,6 +42,9 @@ install:
   - python --version
   - python -c "import struct; print(struct.calcsize('P') * 8)"
 
+  - IF "%PYTHON_ARCH%" == "64" (appveyor DownloadFile "https://dl.dropboxusercontent.com/s/3pdvo56amjjm7b7/win-64.zip" -FileName "C:\windlls.zip") ELSE (appveyor DownloadFile "https://dl.dropboxusercontent.com/s/ky6prqc0zh4wspm/win-32.zip" -FileName "C:\windlls.zip")
+  - 7z x -y "C:\windlls.zip" -o%APPVEYOR_BUILD_FOLDER%\src\urh\dev\native\lib\shared
+
   - pip install -r data\requirements.txt
   - pip install pytest
 

data/build_cx.py

@@ -24,18 +24,49 @@ def build_exe(build_cmd='build'):
 
     if sys.platform == 'win32':
         include_files = [os.path.join("data", 'icons', 'appicon.ico')]
-        arch = "x64" if sys.maxsize > 2 ** 32 else "x86"
-        lib_path = os.path.join("src", "urh", "dev", "native", "lib", "win", arch)
+        lib_path = os.path.join("src", "urh", "dev", "native", "lib", "shared")
         for f in os.listdir(lib_path):
-            include_files.append(os.path.join(lib_path, f))
+            if f.endswith(".dll") or f.endswith(".txt"):
+                include_files.append(os.path.join(lib_path, f))
+
+        shortcut_table = [
+            ("DesktopShortcut",  # Shortcut
+             "DesktopFolder",  # Directory_
+             "Universal Radio Hacker",  # Name
+             "TARGETDIR",  # Component_
+             "[TARGETDIR]urh.exe",  # Target
+             None,  # Arguments
+             None,  # Description
+             None,  # Hotkey
+             None,  # Icon
+             None,  # IconIndex
+             None,  # ShowCmd
+             'TARGETDIR'  # WkDir
+             ),
+
+            ("StartupShortcut",  # Shortcut
+             "StartupFolder",  # Directory_
+             "Universal Radio Hacker",  # Name
+             "TARGETDIR",  # Component_
+             "[TARGETDIR]urh.exe",  # Target
+             None,  # Arguments
+             None,  # Description
+             None,  # Hotkey
+             None,  # Icon
+             None,  # IconIndex
+             None,  # ShowCmd
+             'TARGETDIR'  # WkDir
+             ),
+
+        ]
+
 
         executables = [
             cx_Freeze.Executable(
             app_path,
             targetName="urh.exe",
             icon=os.path.join("data", 'icons', 'appicon.ico'),
-            shortcutName="Universal Radio Hacker",
-            shortcutDir="DesktopFolder",
+            copyright="Copyright (C) 2018 Johannes Pohl, Andreas Noack",
             base="Win32GUI"),
 
             cx_Freeze.Executable(cli_path, targetName="urh_cli.exe")
@@ -47,6 +78,8 @@ def build_exe(build_cmd='build'):
             cx_Freeze.Executable(cli_path, targetName="urh_cli.exe")
         ]
 
+        shortcut_table = None
+
     for f in os.listdir(os.path.join("src", "urh", "dev", "gr", "scripts")):
         if f.endswith(".py"):
             include_files.append(os.path.join("src", "urh", "dev", "gr", "scripts", f))
@@ -69,7 +102,8 @@ def build_exe(build_cmd='build'):
                          'setuptools.msvc'] + plugins
         },
         'bdist_msi': {
-            "upgrade_code": "{96abcdef-1337-4711-cafe-beef4a1ce42}"
+            "upgrade_code": "{96abcdef-1337-4711-cafe-beef4a1ce42}",
+            "data":  {"Shortcut": shortcut_table}
         }
     }
 

data/ui/options.ui

@@ -6,7 +6,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>727</width>
+    <width>773</width>
     <height>749</height>
    </rect>
   </property>
@@ -217,8 +217,8 @@
               <rect>
                <x>0</x>
                <y>0</y>
-               <width>647</width>
-               <height>380</height>
+               <width>98</width>
+               <height>28</height>
               </rect>
              </property>
              <layout class="QVBoxLayout" name="verticalLayout_4"/>
@@ -411,16 +411,6 @@
          </property>
         </widget>
        </item>
-       <item>
-        <widget class="QLabel" name="labelWindowsError">
-         <property name="text">
-          <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;&lt;span style=&quot; color:#ff0000;&quot;&gt;Detected a 32 bit installation of python 3.&lt;/span&gt; Install &lt;span style=&quot; font-weight:600;&quot;&gt;64 bit version&lt;/span&gt; to use native backends.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
-         </property>
-         <property name="wordWrap">
-          <bool>true</bool>
-         </property>
-        </widget>
-       </item>
        <item>
         <widget class="Line" name="line">
          <property name="orientation">
@@ -491,7 +481,8 @@
              <string>View log</string>
             </property>
             <property name="icon">
-             <iconset theme="utilities-log-viewer"/>
+             <iconset theme="utilities-log-viewer">
+              <normaloff>.</normaloff>.</iconset>
             </property>
            </widget>
           </item>

setup.py

@@ -78,20 +78,10 @@ def get_package_data():
         package_data["urh.plugins." + plugin] = ['*.ui', "*.txt"]
 
     package_data["urh.dev.native.lib"] = ["*.cpp", "*.c", "*.pyx", "*.pxd"]
+    package_data["urh.dev.native.include"] = ["*.h"]
 
-    # Bundle headers
-    package_data["urh.dev.native.includes"] = ["*.h"]
-    include_dir = "src/urh/dev/native/includes"
-    for dirpath, dirnames, filenames in os.walk(include_dir):
-        for dir_name in dirnames:
-            rel_dir_path = os.path.relpath(os.path.join(dirpath, dir_name), include_dir)
-            package_data["urh.dev.native.includes."+rel_dir_path.replace(os.sep, ".")] = ["*.h"]
-
-    if sys.platform == "win32" or IS_RELEASE:
-        # we use precompiled device backends on windows
-        # only deploy DLLs on Windows or in release mode to prevent deploying by linux package managers
-        package_data["urh.dev.native.lib.win.x64"] = ["*"]
-        package_data["urh.dev.native.lib.win.x86"] = ["*"]
+    if IS_RELEASE and sys.platform == "win32":
+        package_data["urh.dev.native.lib.shared"] = ["*.dll", "*.txt"]
 
     return package_data
 

src/urh/cli/urh_cli.py

@@ -26,7 +26,7 @@
 
 from urh.util import util
 
-util.set_windows_lib_path()
+util.set_shared_library_path()
 
 try:
     import urh.cythonext.signalFunctions

src/urh/controller/dialogs/OptionsDialog.py

@@ -45,7 +45,6 @@ def __init__(self, installed_plugins, highlighted_plugins=None, parent=None):
 
         # We use bundled native device backends on windows, so no need to reconfigure them
         self.ui.groupBoxNativeOptions.setVisible(sys.platform != "win32")
-        self.ui.labelWindowsError.setVisible(sys.platform == "win32" and platform.architecture()[0] != "64bit")
         self.ui.labelIconTheme.setVisible(sys.platform == "linux")
         self.ui.comboBoxIconTheme.setVisible(sys.platform == "linux")
 
@@ -449,8 +448,8 @@ def on_btn_health_check_clicked(self):
         info = ExtensionHelper.perform_health_check()
         info += "\n" + BackendHandler.perform_soundcard_health_check()
 
-        if util.get_windows_lib_path():
-            info += "\n\n[INFO] Used DLLs from " + util.get_windows_lib_path()
+        if util.get_shared_library_path():
+            info += "\n\n[INFO] Used DLLs from " + util.get_shared_library_path()
 
         d = util.create_textbox_dialog(info, "Health check for native extensions", self)
         d.show()

src/urh/dev/native/Device.py

@@ -7,14 +7,17 @@
 from pickle import UnpicklingError
 
 import numpy as np
-from PyQt5.QtCore import QObject, pyqtSignal
 
 from urh.dev.native.SendConfig import SendConfig
 from urh.util.Logger import logger
 from urh.util.SettingsProxy import SettingsProxy
 
+from urh.util import util
+# set shared library path when processes spawn so they can also find the .so's in bundled case
+util.set_shared_library_path()
 
-class Device(QObject):
+
+class Device(object):
     JOIN_TIMEOUT = 1.0
 
     SYNC_TX_CHUNK_SIZE = 0

src/urh/dev/native/ExtensionHelper.py

@@ -57,16 +57,15 @@ def get_device_extensions(use_cython: bool, library_dirs=None):
     library_dirs = [] if library_dirs is None else library_dirs
 
     cur_dir = os.path.dirname(os.path.realpath(__file__))
-    include_dirs = [os.path.realpath(os.path.join(cur_dir, "includes"))]
-
-    if sys.platform == "win32":
-        if platform.architecture()[0] != "64bit":
-            return []  # only 64 bit python supported for native device backends
+    include_dirs = [os.path.realpath(os.path.join(cur_dir, "include"))]
 
+    if os.path.isdir(os.path.join(cur_dir, "lib/shared")):
+        # Device libs are packaged, so we are in release mode
         result = []
-        lib_dir = os.path.realpath(os.path.join(cur_dir, "lib/win/x64"))
+        include_dirs.append(os.path.realpath(os.path.join(cur_dir, "lib/shared/include")))
+        lib_dir = os.path.realpath(os.path.join(cur_dir, "lib/shared"))
         for dev_name, params in DEVICES.items():
-            # Since windows drivers are bundled we can enforce the macros
+            # Since drivers are bundled we can enforce the macros
             macros = [(extra, None) for extra in params.get("extras", dict())]
             result.append(get_device_extension(dev_name, [params["lib"]], [lib_dir], include_dirs, macros))
 

src/urh/dev/native/includes/libhackrf/hackrf.h → src/urh/dev/native/include/libhackrf/hackrf.h

@@ -8,12 +8,12 @@ All rights reserved.
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
     Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-    Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the
+    Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the 
 	documentation and/or other materials provided with the distribution.
     Neither the name of Great Scott Gadgets nor the names of its contributors may be used to endorse or promote products derived from this software
 	without specific prior written permission.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
 THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
@@ -28,7 +28,7 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSI
 
 #ifdef _WIN32
    #define ADD_EXPORTS
-
+   
   /* You should define ADD_EXPORTS *only* when building the DLL. */
   #ifdef ADD_EXPORTS
     #define ADDAPI __declspec(dllexport)
@@ -125,7 +125,7 @@ struct hackrf_device_list {
 	enum hackrf_usb_board_id *usb_board_ids;
 	int *usb_device_index;
 	int devicecount;
-
+	
 	void **usb_devices;
 	int usb_devicecount;
 };
@@ -147,39 +147,39 @@ extern ADDAPI const char* ADDCALL hackrf_library_release();
 extern ADDAPI hackrf_device_list_t* ADDCALL hackrf_device_list();
 extern ADDAPI int ADDCALL hackrf_device_list_open(hackrf_device_list_t *list, int idx, hackrf_device** device);
 extern ADDAPI void ADDCALL hackrf_device_list_free(hackrf_device_list_t *list);
-
+ 
 extern ADDAPI int ADDCALL hackrf_open(hackrf_device** device);
 extern ADDAPI int ADDCALL hackrf_open_by_serial(const char* const desired_serial_number, hackrf_device** device);
 extern ADDAPI int ADDCALL hackrf_close(hackrf_device* device);
-
+ 
 extern ADDAPI int ADDCALL hackrf_start_rx(hackrf_device* device, hackrf_sample_block_cb_fn callback, void* rx_ctx);
 extern ADDAPI int ADDCALL hackrf_stop_rx(hackrf_device* device);
-
+ 
 extern ADDAPI int ADDCALL hackrf_start_tx(hackrf_device* device, hackrf_sample_block_cb_fn callback, void* tx_ctx);
 extern ADDAPI int ADDCALL hackrf_stop_tx(hackrf_device* device);
 
 /* return HACKRF_TRUE if success */
 extern ADDAPI int ADDCALL hackrf_is_streaming(hackrf_device* device);
-
+ 
 extern ADDAPI int ADDCALL hackrf_max2837_read(hackrf_device* device, uint8_t register_number, uint16_t* value);
 extern ADDAPI int ADDCALL hackrf_max2837_write(hackrf_device* device, uint8_t register_number, uint16_t value);
-
+ 
 extern ADDAPI int ADDCALL hackrf_si5351c_read(hackrf_device* device, uint16_t register_number, uint16_t* value);
 extern ADDAPI int ADDCALL hackrf_si5351c_write(hackrf_device* device, uint16_t register_number, uint16_t value);
-
+ 
 extern ADDAPI int ADDCALL hackrf_set_baseband_filter_bandwidth(hackrf_device* device, const uint32_t bandwidth_hz);
-
+ 
 extern ADDAPI int ADDCALL hackrf_rffc5071_read(hackrf_device* device, uint8_t register_number, uint16_t* value);
 extern ADDAPI int ADDCALL hackrf_rffc5071_write(hackrf_device* device, uint8_t register_number, uint16_t value);
-
+ 
 extern ADDAPI int ADDCALL hackrf_spiflash_erase(hackrf_device* device);
 extern ADDAPI int ADDCALL hackrf_spiflash_write(hackrf_device* device, const uint32_t address, const uint16_t length, unsigned char* const data);
 extern ADDAPI int ADDCALL hackrf_spiflash_read(hackrf_device* device, const uint32_t address, const uint16_t length, unsigned char* data);
 
 /* device will need to be reset after hackrf_cpld_write */
 extern ADDAPI int ADDCALL hackrf_cpld_write(hackrf_device* device,
 		unsigned char* const data, const unsigned int total_length);
-
+		
 extern ADDAPI int ADDCALL hackrf_board_id_read(hackrf_device* device, uint8_t* value);
 extern ADDAPI int ADDCALL hackrf_version_string_read(hackrf_device* device, char* version, uint8_t length);
 extern ADDAPI int ADDCALL hackrf_usb_api_version_read(hackrf_device* device, uint16_t* version);