diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e3fd6f08f2f..984c13812d92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -106,7 +106,7 @@ jobs: maxminddb: "--with-maxminddb" msan: "--with-sanitizer" nBPF: "" - lto_gold_linker: "--with-lto-and-gold-linker" + lto_gold_linker: "" - compiler: "clang-9" # "Oldest" clang easily available os: ubuntu-20.04 arch: "x86_64" @@ -125,22 +125,34 @@ jobs: maxminddb: "--with-maxminddb" msan: "--with-sanitizer" nBPF: "" + lto_gold_linker: "" + - compiler: "gcc-14" # "Newest" gcc easily available with lto/gold linker + os: ubuntu-24.04 + arch: "x86_64" + gcrypt: "" + pcre: "--with-pcre2" + maxminddb: "--with-maxminddb" + msan: "" + nBPF: "" lto_gold_linker: "--with-lto-and-gold-linker" - - compiler: "cc" - os: ubuntu-latest + - compiler: "clang-18" # "Newest" clang easily available with lto/gold linker + ar: "llvm-ar-18" + ranlib: "llvm-ranlib-18" + os: ubuntu-24.04 arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-thread-sanitizer" + msan: "" nBPF: "" + lto_gold_linker: "--with-lto-and-gold-linker" - compiler: "cc" os: ubuntu-latest arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-sanitizer" + msan: "" nBPF: "nBPF" - compiler: "cc" os: ubuntu-latest @@ -148,11 +160,11 @@ jobs: gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-sanitizer" + msan: "" nBPF: "" global_context: "--disable-global-context-support" - compiler: "clang" #TODO: some issues with masan/clang/ubuntu-24.04 - os: ubuntu-22.04 + os: ubuntu-24.04 arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" @@ -222,7 +234,7 @@ jobs: sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev sudo apt-get install rrdtool librrd-dev parallel - name: Install Ubuntu Prerequisites [Mingw-w64] (runs only on ubuntu jobs) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.lto_gold_linker, '--with-') #Only on a few "standard" builds run: | sudo apt-get install gcc-mingw-w64 libc6-dev - name: Install Ubuntu Prerequisites (libgcrypt) @@ -318,15 +330,20 @@ jobs: DESTDIR=/tmp/ndpi make install ls -alhHR /tmp/ndpi - name: Test nDPI [SYMBOLS] - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.lto_gold_linker, '--with-') #Only on a few "standard" builds + run: | ./utils/check_symbols.sh || { FAILED=$?; echo "::error file=${NDPI_LIB}::Unwanted libc symbols found: ${FAILED}. Please make sure to use only ndpi_malloc/ndpi_calloc/ndpi_realloc/ndpi_free wrapper instead of malloc/calloc/realloc/free."; false; } env: NDPI_LIB: src/lib/libndpi.a - name: Test nDPI [DIFF] - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') + if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer') run: | NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh + - name: Test nDPI (quick) [DIFF] + if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.msan, '--with-memory-sanitizer') + run: | + NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh - name: Test nDPI [UNIT] #Some issues with masan + json-c. Disable the test as workaround if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') @@ -361,7 +378,7 @@ jobs: make cd ../.. - name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.lto_gold_linker, '--with-') #Only on a few "standard" builds run: | make distclean ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32 @@ -387,10 +404,10 @@ jobs: apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && git config --global --add safe.directory $(realpath .) && env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && - make -j all && + make -j $(nproc) all && make -C example ndpiSimpleIntegration && make -C rrdtool && - make check VERBOSE=1 + NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 make check VERBOSE=1 " - name: Display qemu specified architecture (armhf - little endian) if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf') @@ -411,10 +428,10 @@ jobs: apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && git config --global --add safe.directory $(realpath .) && env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && - make -j all && + make -j $(nproc) all && make -C example ndpiSimpleIntegration && make -C rrdtool && - make check VERBOSE=1 + NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 make check VERBOSE=1 " - name: Display qemu specified architecture (s390x - big endian) if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x') @@ -435,8 +452,8 @@ jobs: apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && git config --global --add safe.directory $(realpath .) && env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && - make -j all && + make -j $(nproc) all && make -C example ndpiSimpleIntegration && make -C rrdtool && - make check VERBOSE=1 + NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 make check VERBOSE=1 " diff --git a/.github/workflows/build_scheduled.yml b/.github/workflows/build_scheduled.yml index 962da97152b8..c146d3599843 100644 --- a/.github/workflows/build_scheduled.yml +++ b/.github/workflows/build_scheduled.yml @@ -109,3 +109,26 @@ jobs: name: ndpi-performance path: ndpi-performance-upload retention-days: 7 + + thread-sanitizer: + name: Thread-Sanitizer (ubuntu-latest) + runs-on: ubuntu-latest + env: + CFLAGS: -Werror + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install Ubuntu Prerequisites + run: | + sudo apt-get update + sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libpcap-dev libjson-c-dev rrdtool librrd-dev parallel + - name: Configure nDPI library + run: | + ./autogen.sh --enable-option-checking=fatal --with-thread-sanitizer + - name: Build nDPI library + run: | + make -j all + - name: Test nDPI + run: | + NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index de65fccb6c0b..28cfc9d0a925 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -18,7 +18,7 @@ jobs: uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master with: oss-fuzz-project-name: 'ndpi' - fuzz-seconds: 1200 + fuzz-seconds: 600 dry-run: false sanitizer: ${{ matrix.sanitizer }} - name: Check Crash (fails when a crash is detected) diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 61989839fa73..fa0e779b4163 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -4015,7 +4015,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us /* LRU caches */ for(i = 0; i < NDPI_LRUCACHE_MAX; i++) { - struct ndpi_lru_cache_stats s; + struct ndpi_lru_cache_stats s = { 0 }; int scope; char param[64]; @@ -4035,7 +4035,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us /* Automas */ for(i = 0; i < NDPI_AUTOMA_MAX; i++) { - struct ndpi_automa_stats s; + struct ndpi_automa_stats s = { 0 }; ndpi_get_automa_stats(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, &s); cumulative_stats.automa_stats[i].n_search += s.n_search; cumulative_stats.automa_stats[i].n_found += s.n_found; @@ -4043,7 +4043,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us /* Patricia trees */ for(i = 0; i < NDPI_PTREE_MAX; i++) { - struct ndpi_patricia_tree_stats s; + struct ndpi_patricia_tree_stats s = { 0 }; ndpi_get_patricia_stats(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, &s); cumulative_stats.patricia_stats[i].n_search += s.n_search; cumulative_stats.patricia_stats[i].n_found += s.n_found; diff --git a/tests/do.sh.in b/tests/do.sh.in index 93822576b149..ae6dcc97699f 100755 --- a/tests/do.sh.in +++ b/tests/do.sh.in @@ -31,6 +31,11 @@ if [ "${NDPI_SKIP_PARALLEL_BAR}" = "1" ]; then SKIP_PARALLEL_BAR=1 fi +TEST_ONLY_RECENTLY_UPDATED_PCAPS=0 +if [ "${NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS}" = "1" ]; then + TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 +fi + #Remember: valgrind and *SAN are incompatible! CMD_PREFIX="${CMD_PREFIX}" if [ "${NDPI_TESTS_WINE}" = "1" ]; then @@ -239,10 +244,16 @@ for d in $(find ./cfgs/* -type d -maxdepth 0 2>/dev/null) ; do cd ./cfgs/"$(basename "$d")" || exit 1 - if [ "$#" -ne 0 ]; then - PCAPS=$* + if [ $TEST_ONLY_RECENTLY_UPDATED_PCAPS -eq 1 ]; then + # Latest 10 pcaps updatetd recently (to be more precise, pcap which triggered a change + # in their result lately) + PCAPS=$(cd result || exit 1; find * -type f -print0 | xargs -0 ls -t | head -10 | xargs basename -s .out) else - PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*) + if [ "$#" -ne 0 ]; then + PCAPS=$* + else + PCAPS=$(cd pcap || exit 1; /bin/ls -- *.*cap*) + fi fi FAILURES=() READER_EXTRA_PARAM="" diff --git a/utils/check_symbols.sh b/utils/check_symbols.sh index ec23bfa9065d..31d1544d22f0 100755 --- a/utils/check_symbols.sh +++ b/utils/check_symbols.sh @@ -28,7 +28,7 @@ for line in $(nm -P -u "${NDPI_LIB}"); do 'printf'|'fprintf') SKIP=1 ;; esac ;; - '[ahocorasick.o]') + '[ahocorasick.o]'|'[ndpi_cache.o]') case "${FOUND_SYMBOL}" in 'fprintf') SKIP=1 ;; esac