Update some CI jobs

* Move ThreadSanitizer job to the scheduled jobs (once a day): all our tests are intrinsically mono-thread and this job takes quite some time * Two explicit jobs to test LTO and Gold linker, used by oss-fuzz * Two explicit jobs for Windows (with msys2) * Run address sanitizer only on the 4 main jobs: newest/oldest gcc/clang * Reduce the time used by fuzzing jobs. Note that oss-fuzz is continuosly fuzzing our code! This way, the main matrix is a little bit simpler
ntop · Jan 7, 2025 · 9dfcd4a · 9dfcd4a
1 parent c3d19be
commit 9dfcd4a
Show file tree

Hide file tree

Showing 4 changed files with 119 additions and 57 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -43,6 +43,7 @@ jobs:
         run: |
           cd python
           python tests.py
+
   test-scripts:
     name: Test Utils (ubuntu-latest)
     runs-on: ubuntu-latest
@@ -67,7 +68,7 @@ jobs:
           git diff-index --quiet HEAD -- || true
 
   test:
-    name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.gcrypt }} ${{ matrix.compiler }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }} ${{ matrix.nBPF }} ${{matrix.lto_gold_linker}} ${{matrix.global_context}}
+    name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.gcrypt }} ${{ matrix.compiler }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }} ${{ matrix.nBPF }} ${{matrix.global_context}}
     runs-on: ${{ matrix.os }}
     env:
       CC: ${{ matrix.compiler }}
@@ -77,17 +78,14 @@ jobs:
       matrix:
         # macOS-latest == macos-14 on **ARM64**. Also macos-15 is on arm64
         # There are some issues with external dependencies on macOS-14/15. Disable it for the time being
-        os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04", "macOS-13", "windows-latest"]
+        os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04", "macOS-13", "macOS-14"]
         arch: ["x86_64"]
         gcrypt: ["--with-local-libgcrypt", ""]
         compiler: ["cc"]
-        ar: ["ar"]
-        ranlib: ["ranlib"]
         pcre: [""]
         maxminddb: [""]
         msan: [""]
         nBPF: [""]
-        lto_gold_linker: [""]
         global_context: [""] #Enable by default
         include:
           - compiler: "gcc-4.9" # "Oldest" gcc easily available. To simulate RHEL7
@@ -106,7 +104,6 @@ jobs:
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
-            lto_gold_linker: "--with-lto-and-gold-linker"
           - compiler: "clang-9" # "Oldest" clang easily available
             os: ubuntu-20.04
             arch: "x86_64"
@@ -116,39 +113,28 @@ jobs:
             msan: "--with-sanitizer"
             nBPF: ""
           - compiler: "clang-18" # "Newest" clang easily available. See also below...
-            ar: "llvm-ar-18"
-            ranlib: "llvm-ranlib-18"
             os: ubuntu-24.04
             arch: "x86_64"
             gcrypt: ""
             pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
-            lto_gold_linker: "--with-lto-and-gold-linker"
           - compiler: "cc"
             os: ubuntu-latest
             arch: "x86_64"
             gcrypt: ""
             pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
-            msan: "--with-thread-sanitizer"
-            nBPF: ""
-          - compiler: "cc"
-            os: ubuntu-latest
-            arch: "x86_64"
-            gcrypt: ""
-            pcre: "--with-pcre2"
-            maxminddb: "--with-maxminddb"
-            msan: "--with-sanitizer"
+            msan: ""
             nBPF: "nBPF"
           - compiler: "cc"
             os: ubuntu-latest
             arch: "x86_64"
             gcrypt: ""
             pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
-            msan: "--with-sanitizer"
+            msan: ""
             nBPF: ""
             global_context: "--disable-global-context-support"
           - compiler: "clang" #TODO: some issues with masan/clang/ubuntu-24.04
@@ -254,13 +240,6 @@ jobs:
           sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5
           sudo apt-get update
           sudo apt-get install ${{ matrix.compiler }}
-      - name: Install Windows msys2 prerequisites
-        if: startsWith(matrix.os, 'windows')
-        uses: msys2/setup-msys2@v2
-        with:
-          msystem: MINGW64
-          update: true
-          install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel
       - name: Installing MacOS prerequisites
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64')
         run: |
@@ -269,6 +248,7 @@ jobs:
       - name: Install MacOS Prerequisites (libgcrypt)
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
         run: |
+          echo ${HOMEBREW_DIR}
           brew install libgcrypt
       - name: Install MacOS Prerequisites (libpcre2)
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2')
@@ -281,22 +261,13 @@ jobs:
       - name: Configure nDPI on Ubuntu
         if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64')
         run: |
-          AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{matrix.lto_gold_linker}} ${{matrix.global_context}}
+          ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{matrix.global_context}}
       - name: Configure nDPI on MacOS
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc')
         run: |
           ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs
-      - name: Configure nDPI on Windows msys2
-        if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc')
-        run: |
-          msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap ${{ matrix.gcrypt }}'
-      - name: Build nDPI on Windows msys2
-        if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc')
-        run: |
-          msys2 -c 'make -j all'
-          msys2 -c 'ldd ./example/ndpiReader.exe'
       - name: Build nDPI
-        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15')
+        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15')
         run: |
           make -j all
           make -C example ndpiSimpleIntegration
@@ -309,45 +280,33 @@ jobs:
           #There are somes issues with librrd
           #make -C rrdtool
       - name: Print nDPI long help
-        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
+        if: startsWith(matrix.arch, 'x86_64')
         run: |
           cd ./example && ./ndpiReader -H
       - name: Install nDPI
-        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
+        if: startsWith(matrix.arch, 'x86_64')
         run: |
           DESTDIR=/tmp/ndpi make install
           ls -alhHR /tmp/ndpi
       - name: Test nDPI [SYMBOLS]
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers
+        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') #Only on a few "standard" builds
         run: |
           ./utils/check_symbols.sh || { FAILED=$?; echo "::error file=${NDPI_LIB}::Unwanted libc symbols found: ${FAILED}. Please make sure to use only ndpi_malloc/ndpi_calloc/ndpi_realloc/ndpi_free wrapper instead of malloc/calloc/realloc/free."; false; }
         env:
           NDPI_LIB: src/lib/libndpi.a
       - name: Test nDPI [DIFF]
-        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
+        if: startsWith(matrix.arch, 'x86_64')
         run: |
           NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
       - name: Test nDPI [UNIT]
         #Some issues with masan + json-c. Disable the test as workaround
-        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15')
+        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15')
         run: |
           ./tests/do-unit.sh
       - name: Test nDPI [DGA]
-        if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
+        if: startsWith(matrix.arch, 'x86_64')
         run: |
           ./tests/do-dga.sh
-      - name: Test nDPI [DIFF] (runs only on windows jobs)
-        if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows')
-        run: |
-          msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh'
-      - name: Test nDPI [UNIT] (runs only on windows jobs)
-        if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows')
-        run: |
-          msys2 -c './tests/do-unit.sh'
-      - name: Test nDPI [DGA] (runs only on windows jobs)
-        if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows')
-        run: |
-          msys2 -c './tests/do-dga.sh'
       - name: Generate/Verify tarball
         if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64')
         run: |
@@ -361,7 +320,7 @@ jobs:
           make
           cd ../..
       - name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs)
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers
+        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') #Only on a few "standard" builds
         run: |
           make distclean
           ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32
@@ -440,3 +399,74 @@ jobs:
             make -C rrdtool &&
             make check VERBOSE=1
             "
+
+  test-lto-gold:
+    # Options used by oss-fuzz: we only want to check that everything compile fine; no need to run the tests
+    name: LTO and Gold Linker ${{ matrix.compiler }}
+    runs-on: ubuntu-24.04
+    env:
+      CC: ${{ matrix.compiler }}
+      CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS
+    strategy:
+      fail-fast: true
+      matrix:
+        compiler: ["gcc-14", "clang-18"] # "Newest" gcc/clang easily available
+        include:
+          - compiler: "gcc-14"
+            ar: "ar"
+            ranlib: "ranlib"
+          - compiler: "clang-18"
+            ar: "llvm-ar-18"
+            ranlib: "llvm-ranlib-18"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Install Ubuntu Prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev parallel
+      - name: Configure nDPI
+        run: |
+          AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --with-sanitizer --with-lto-and-gold-linker
+      - name: Build nDPI
+        run: |
+          make -j $(nproc) all
+          make -j $(nproc) -C example ndpiSimpleIntegration
+          make -j $(nproc) -C rrdtool
+      - name: Print nDPI long help
+        run: |
+          cd ./example && ./ndpiReader -H
+
+  test-windows:
+    name: ${{ matrix.os }} (msys2) ${{ matrix.gcrypt }}
+    runs-on: ${{ matrix.os }}
+    env:
+      CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS
+    strategy:
+      fail-fast: true
+      matrix:
+        os: ["windows-latest"]
+        gcrypt: ["--with-local-libgcrypt", ""]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
+      - name: Install Windows msys2 prerequisites
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: MINGW64
+          update: true
+          install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel
+      - name: Configure nDPI on Windows msys2
+        run: |
+          msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap ${{ matrix.gcrypt }}'
+      - name: Build nDPI on Windows msys2
+        run: |
+          msys2 -c 'make -j all'
+          msys2 -c 'ldd ./example/ndpiReader.exe'
+      - name: Tests
+        run: |
+          msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh'
+          msys2 -c './tests/do-unit.sh'
+          msys2 -c './tests/do-dga.sh'
diff --git a/.github/workflows/build_scheduled.yml b/.github/workflows/build_scheduled.yml
@@ -109,3 +109,28 @@ jobs:
           name: ndpi-performance
           path: ndpi-performance-upload
           retention-days: 7
+
+  threadsanitizer:
+    name: Thread Sanitizer (ubuntu-latest)
+    runs-on: ubuntu-latest
+    env:
+      CFLAGS: -Wextra -Werror
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install Ubuntu Prerequisites
+        run: |
+          sudo apt-get update
+          sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c libpcap-dev rrdtool librrd-dev parallel
+      - name: Configure nDPI
+        run: |
+          ./autogen.sh --enable-option-checking=fatal --with-thread-sanitizer
+      - name: Build nDPI
+        run: |
+          make -j $(nproc) all
+          make -j $(nproc) -C example ndpiSimpleIntegration
+          make -j $(nproc) -C rrdtool
+      - name: Tests
+        run: |
+          NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml
@@ -18,7 +18,7 @@ jobs:
         uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
         with:
           oss-fuzz-project-name: 'ndpi'
-          fuzz-seconds: 1200
+          fuzz-seconds: 600
           dry-run: false
           sanitizer: ${{ matrix.sanitizer }}
       - name: Check Crash (fails when a crash is detected)

diff --git a/configure.ac b/configure.ac
@@ -228,6 +228,13 @@ AS_IF([test "x${enable_tls_sigs}" = "xyes"],[
 
 NDPI_CFLAGS="-W -Wall -Wno-address-of-packed-member ${NDPI_CFLAGS}"
 
+dnl> MacOS brew.sh
+#HOMEBREW_DIR=/opt/homebrew
+if test -d ${HOMEBREW_DIR}; then
+ NDPI_CFLAGS="${NDPI_CFLAGS} -I ${HOMEBREW_DIR}/include"
+ NDPI_LDFLAGS="${NDPI_LDFLAGS} -L ${HOMEBREW_DIR}/lib"
+fi
+
 AS_IF([test "${with_lto_and_gold_linker+set}" = set], [
        NDPI_CFLAGS="${NDPI_CFLAGS} -flto -fuse-ld=gold -Wno-unused-command-line-argument"
        NDPI_LDFLAGS="${NDPI_LDFLAGS} ${NDPI_CFLAGS}"