GnuRadio service (#123)

* Implement GNU Radio acquisition and flow graph service * CI: Add gr-digitizers picoscope dependencies * Remove old acquisition and flowgraph mock workers * Enable unit testing * CI: Update to gcc13 and emscripten 3.1.4, Enable C++23, move to latest opencmw-cpp/graph-prototype * UI: Split default dashboard .ddd file into text components Use text files instead of .ddd here, to make it easier to edit and to have readable diffs. * Move acquisition and flow graph data structures to own lib Move them to a custom lib, to share them with the UI, and among different workers. * Improve emscripten support Enable threading and exceptions for emscripten. Add mimetypes and cors headers to the rest endpoint. Add the worker javascript to the rest endpoint. * Simplify poller handling The issue of the datasink creating pollers with finished=false after stop has been fixed in graph-prototype, so the logic here can be simplified. --------- Co-authored-by: Alexander Krimm <[email protected]>
fair-acc · Nov 13, 2023 · 7afc207 · 7afc207
1 parent 746473b
commit 7afc207
Show file tree

Hide file tree

Showing 32 changed files with 1,745 additions and 640 deletions.
diff --git a/.github/workflows/build_cmake.yml b/.github/workflows/build_cmake.yml
@@ -21,14 +21,14 @@ jobs:
       fail-fast: false
       matrix:
         configurations:
-          - name: Ubuntu Latest gcc
+          - name: Ubuntu gcc 13
             os: ubuntu-22.04
-            compiler: gcc
+            compiler: gcc13
           # - name: Ubuntu Latest clang
           #   os: ubuntu-22.04 # pre-release, ubuntu-latest still points to ubuntu-2004
           #   compiler: clang
         # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
-        cmake-build-type: [ Release ] #, Debug ]
+        cmake-build-type: [ Release, Debug ]
 
     steps:
     - uses: actions/checkout@v3
@@ -43,39 +43,76 @@ jobs:
         path: ${{runner.workspace}}/build/_deps
         key: ${{ runner.os }}-${{ matrix.configurations.compiler }}-${{ matrix.cmake-build-type }}-${{ hashFiles('CMakeLists.txt') }}-${{ hashFiles('cmake/Dependencies.cmake') }}
 
-    - name: Setup emsdk
-      uses: mymindstorm/setup-emsdk@v12
-      with:
-        version: 3.1.41
-        actions-cache-folder: 'emsdk-cache'
+    - name: Install emscripten
+      shell: bash
+      run: |
+        cd
+        git clone --depth=1 https://github.com/emscripten-core/emsdk.git
+        cd emsdk
+        # Download and install emscripten.
+        ./emsdk install 3.1.44 # (latest = 3.1.47 -> 21/10/2023)
+        # Make "active" for the current user. (writes .emscripten file)
+        ./emsdk activate 3.1.44
 
-    - name: Install gcc-12
-      if: matrix.configurations.compiler == 'gcc'
+    - name: Install gcovr
+      shell: bash
+      if: matrix.cmake-build-type == 'Debug'
       run: |
-        sudo apt update
-        sudo apt install -y gcc-12 g++-12
-        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 110 --slave /usr/bin/g++ g++ /usr/bin/g++-12 --slave /usr/bin/gcov gcov /usr/bin/gcov-12
+        python3 -m pip install gcovr --user --no-warn-script-location
+        gcovr --version
+
+    - name: Install gcc-13
+      run: |
+        sudo apt-get install -y gcc-13 g++-13 # gcovr # packaged gcovr is too old for gcc13
+        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 110 --slave /usr/bin/g++ g++ /usr/bin/g++-13 --slave /usr/bin/gcov gcov /usr/bin/gcov-13
 
     - name: Install openGL
       run: |
         sudo apt update
         sudo apt install -y libx11-dev libgl1-mesa-dev libsdl2-dev
 
+    # Temporary dependency, until we rely on the plugin system to load gr-digitizers blocks at runtime
+    - name: Install picoscope libraries
+      run: |
+        # https://www.picotech.com/downloads/linux
+        wget -O - https://labs.picotech.com/Release.gpg.key|sudo apt-key add -
+        sudo add-apt-repository 'deb https://labs.picotech.com/rc/picoscope7/debian/ picoscope main'
+        sudo apt update
+        sudo apt install -y udev libusb-1.0-0-dev libps3000a libps4000a libps5000a libps6000 libps6000a || true # ignore udev errors in post install because of udev in container
+
+
     - name: Configure CMake
       shell: bash
-      run: cmake -S . -B ../build -DCMAKE_BUILD_TYPE=${{ matrix.cmake-build-type }} -DEMCMAKE_COMMAND=`which emcmake`
+      run: |
+        export SYSTEM_NODE=`which node` # use system node instead of old version distributed with emsdk for threading support
+        source ~/emsdk/emsdk_env.sh
+        cmake -S . -B ../build -DCMAKE_BUILD_TYPE=${{ matrix.cmake-build-type }} -DEMCMAKE_COMMAND=`which emcmake`
 
     - name: Build
       shell: bash
-      run: cmake --build ../build
+      run: |
+        source ~/emsdk/emsdk_env.sh
+        cmake --build ../build
+
+    - name: execute tests
+      if: matrix.configurations.compiler != 'gcc13' || matrix.cmake-build-type != 'Debug'
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      run: ctest --output-on-failure
+
+    - name: execute tests with coverage
+      if: matrix.configurations.compiler == 'gcc13' && matrix.cmake-build-type == 'Debug'
+      working-directory: ${{runner.workspace}}/build
+      shell: bash
+      run: cmake --build . --target opendigitizer_coverage
 
     - name: Setup Pages
       uses: actions/configure-pages@v3
     - name: Upload artifact
       uses: actions/upload-pages-artifact@v1
       with:
         path: '../build/CMakeExternals/Build/ui-wasm/web/'
-        
+
   deploy_pages:
     name: Deploy to GitHub Pages
     if: ${{ github.ref_name == 'main' && github.event_name == 'push' }}

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -2,33 +2,41 @@ cmake_minimum_required(VERSION 3.12)
 
 project(opendigitizer CXX)
 
-set(EMCMAKE_COMMAND "emcmake" CACHE FILEPATH "Path to the emcmake command")
+set(EMCMAKE_COMMAND
+    "emcmake"
+    CACHE FILEPATH "Path to the emcmake command")
 
 include(ExternalProject)
 
 set(PREFIX_DIR ${CMAKE_BINARY_DIR}/CMakeExternals/Prefix)
 set(BUILD_DIR ${CMAKE_BINARY_DIR}/CMakeExternals/Build)
 set(INSTALL_DIR ${CMAKE_BINARY_DIR}/CMakeExternals/Install)
 
-
+add_subdirectory(src/acquisition)
 add_subdirectory(src/ui)
 
-ExternalProject_Add( ui-wasm
+ExternalProject_Add(
+  ui-wasm
   SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/ui
   PREFIX ${PREFIX_DIR}/ui-wasm
-        CONFIGURE_COMMAND ${EMCMAKE_COMMAND} ${CMAKE_COMMAND} -G "Unix Makefiles" -S ${CMAKE_SOURCE_DIR}/src/ui -B ${BUILD_DIR}/ui-wasm -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_MAKE_PROGRAM=${CMAKE_SOURCE_DIR}/emmake_make.sh
+  CONFIGURE_COMMAND
+    ${EMCMAKE_COMMAND} ${CMAKE_COMMAND} -G "Unix Makefiles" -S ${CMAKE_SOURCE_DIR}/src/ui -B ${BUILD_DIR}/ui-wasm
+    -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_MAKE_PROGRAM=${CMAKE_SOURCE_DIR}/emmake_make.sh
   BUILD_COMMAND cmake --build ${BUILD_DIR}/ui-wasm
   BINARY_DIR ${BUILD_DIR}/ui-wasm
   BUILD_ALWAYS 1
   INSTALL_COMMAND "" # skip install
-  CMAKE_ARGS
-    -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE}
-    -DCMAKE_MAKE_PROGRAM:FILEPATH=${CMAKE_MAKE_PROGRAM}
+  CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DCMAKE_MAKE_PROGRAM:FILEPATH=${CMAKE_MAKE_PROGRAM}
   BUILD_BYPRODUCTS
     ${BUILD_DIR}/ui-wasm/web/index.html
     ${BUILD_DIR}/ui-wasm/web/index.js
-    ${BUILD_DIR}/ui-wasm/web/index.wasm
-)
+    ${BUILD_DIR}/ui-wasm/web/index.worker.js
+    ${BUILD_DIR}/ui-wasm/web/index.wasm)
 set(WASM_BUILD_DIR ${BUILD_DIR}/ui-wasm)
 
+option(OPENDIGITIZER_ENABLE_TESTING "Enable unit tests" ON)
+if(OPENDIGITIZER_ENABLE_TESTING)
+  enable_testing()
+endif()
+
 add_subdirectory(src/service)
diff --git a/src/acquisition/CMakeLists.txt b/src/acquisition/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_library(od_acquisition INTERFACE)
+
+target_include_directories(od_acquisition INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
+                                                    $<INSTALL_INTERFACE:include/>)
+
+target_link_libraries(od_acquisition INTERFACE project_options project_warnings)
+set_target_properties(od_acquisition PROPERTIES PUBLIC_HEADER daq_api.hpp)
diff --git a/src/service/acquisition/daq_api.hpp → src/acquisition/daq_api.hpp b/src/service/acquisition/daq_api.hpp → src/acquisition/daq_api.hpp
@@ -1,4 +1,5 @@
-#pragma once
+#ifndef OPENDIGITIZER_ACQUISITION_DAQ_API_H
+#define OPENDIGITIZER_ACQUISITION_DAQ_API_H
 
 #include <MultiArray.hpp>
 #include <opencmw.hpp>
@@ -7,6 +8,24 @@
 #include <units/isq/si/time.h>
 #include <vector>
 
+namespace opendigitizer::flowgraph {
+
+struct FilterContext {
+    opencmw::MIME::MimeType contentType = opencmw::MIME::JSON;
+};
+
+struct Flowgraph {
+    std::string flowgraph;
+    std::string layout;
+};
+
+} // namespace opendigitizer::flowgraph
+
+ENABLE_REFLECTION_FOR(opendigitizer::flowgraph::FilterContext, contentType)
+ENABLE_REFLECTION_FOR(opendigitizer::flowgraph::Flowgraph, flowgraph, layout)
+
+namespace opendigitizer::acq {
+
 using opencmw::Annotated;
 using namespace units::isq;
 using namespace units::isq::si;
@@ -34,7 +53,6 @@ struct Acquisition {
     Annotated<float, opencmw::NoUnit, "minimum expected value for channel/signal">               channelRangeMax;
     Annotated<float, opencmw::NoUnit, "temperature of the measurement device">                   temperature;
 };
-ENABLE_REFLECTION_FOR(Acquisition, selectedFilter, acqTriggerName, acqTriggerTimeStamp, acqLocalTimeStamp, channelTimeBase, channelUserDelay, channelActualDelay, channelName, channelValue, channelError, channelUnit, status, channelRangeMin, channelRangeMax, temperature)
 
 /**
  * Generic frequency domain data object.
@@ -57,25 +75,35 @@ struct AcquisitionSpectra {
     Annotated<std::vector<long>, si::time<si::second>, "timestamps of samples">                  channelPhase_dim1_labels; // todo: either nanosecond or float
     Annotated<std::vector<float>, opencmw::NoUnit, "freqency scale">                             channelPhase_dim2_labels; // unit: Hz or f_rev
 };
-ENABLE_REFLECTION_FOR(AcquisitionSpectra, selectedFilter, acqTriggerName, acqTriggerTimeStamp, acqLocalTimeStamp, channelName, channelMagnitude, channelMagnitude_dimensions, channelMagnitude_labels, channelMagnitude_dim1_labels, channelMagnitude_dim2_labels, channelPhase, channelPhase_labels, channelPhase_dim1_labels, channelPhase_dim2_labels)
 // clang-format: ON
 
 struct TimeDomainContext {
-    std::string             channelNameFilter;
-    int32_t                 acquisitionModeFilter = 0; // STREAMING
-    std::string             triggerNameFilter;
-    int32_t                 maxClientUpdateFrequencyFilter = 25;
-    opencmw::MIME::MimeType contentType                    = opencmw::MIME::JSON;
+    std::string channelNameFilter;
+    std::string acquisitionModeFilter = "continuous"; // one of "continuous", "triggered", "multiplexed", "snapshot"
+    std::string triggerNameFilter;
+    int32_t     maxClientUpdateFrequencyFilter = 25;
+    // TODO should we use sensible defaults for the following properties?
+    // TODO make the following unsigned? (add unsigned support to query serialiser)
+    int32_t                 preSamples        = 0;                     // Trigger mode
+    int32_t                 postSamples       = 0;                     // Trigger mode
+    int32_t                 maximumWindowSize = 65535;                 // Multiplexed mode
+    int64_t                 snapshotDelay     = 0;                     // nanoseconds, Snapshot mode
+    opencmw::MIME::MimeType contentType       = opencmw::MIME::BINARY; // YaS
 };
 
-ENABLE_REFLECTION_FOR(TimeDomainContext, channelNameFilter, acquisitionModeFilter, triggerNameFilter, maxClientUpdateFrequencyFilter, contentType)
-
 struct FreqDomainContext {
     std::string             channelNameFilter;
-    int32_t                 acquisitionModeFilter = 0; // STREAMING
+    std::string             acquisitionModeFilter = "continuous"; // one of "continuous", "triggered", "multiplexed", "snapshot"
     std::string             triggerNameFilter;
     int32_t                 maxClientUpdateFrequencyFilter = 25;
-    opencmw::MIME::MimeType contentType                    = opencmw::MIME::JSON;
+    opencmw::MIME::MimeType contentType                    = opencmw::MIME::BINARY; // YaS
 };
 
-ENABLE_REFLECTION_FOR(FreqDomainContext, channelNameFilter, acquisitionModeFilter, triggerNameFilter, maxClientUpdateFrequencyFilter, contentType)
+} // namespace opendigitizer::acq
+
+ENABLE_REFLECTION_FOR(opendigitizer::acq::Acquisition, selectedFilter, acqTriggerName, acqTriggerTimeStamp, acqLocalTimeStamp, channelTimeBase, channelUserDelay, channelActualDelay, channelName, channelValue, channelError, channelUnit, status, channelRangeMin, channelRangeMax, temperature)
+ENABLE_REFLECTION_FOR(opendigitizer::acq::AcquisitionSpectra, selectedFilter, acqTriggerName, acqTriggerTimeStamp, acqLocalTimeStamp, channelName, channelMagnitude, channelMagnitude_dimensions, channelMagnitude_labels, channelMagnitude_dim1_labels, channelMagnitude_dim2_labels, channelPhase, channelPhase_labels, channelPhase_dim1_labels, channelPhase_dim2_labels)
+ENABLE_REFLECTION_FOR(opendigitizer::acq::TimeDomainContext, channelNameFilter, acquisitionModeFilter, triggerNameFilter, maxClientUpdateFrequencyFilter, preSamples, postSamples, maximumWindowSize, snapshotDelay, contentType)
+ENABLE_REFLECTION_FOR(opendigitizer::acq::FreqDomainContext, channelNameFilter, acquisitionModeFilter, triggerNameFilter, maxClientUpdateFrequencyFilter, contentType)
+
+#endif