Move from PCRE to PCRE2 (#2134)

Move from PCRE to PCRE2. PCRE is EOL and won't receive any security
updates anymore. Convert to PCRE2 by converting any function PCRE2 new
API.

Also update every entry in github workflows and README to point to the
new configure flag. (--with-pcre2)

Signed-off-by: Christian Marangi <[email protected]>

.github/ISSUE_TEMPLATE/bug_template.md

@@ -21,7 +21,7 @@ A clear and concise description of what happening.
 * OS version: [e.g. 18.04]
 * Architecture: [e.g. arm64]
 * nDPI version or commit hash: [e.g. 4.0-stable or 937357e4bc55610f116f66d15a8e0fc1e260c02c].
-* nDPI compilation flags used: if you are building from source [e.g. --with-pcre --with-local-libgcrypt].
+* nDPI compilation flags used: if you are building from source [e.g. --with-pcre2 --with-local-libgcrypt].
 * Attach the `config.log` file generated after `./configure` ran (if you are building from source).
 
 ## How to reproduce the reported bug

.github/workflows/build.yml

@@ -93,15 +93,15 @@ jobs:
             os: ubuntu-20.04
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
           - compiler: "gcc-12" # "Newest" gcc easily available
             os: ubuntu-22.04
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
@@ -110,7 +110,7 @@ jobs:
             os: ubuntu-20.04
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
@@ -120,7 +120,7 @@ jobs:
             os: ubuntu-22.04
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
@@ -129,63 +129,63 @@ jobs:
             os: ubuntu-latest
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-thread-sanitizer"
             nBPF: ""
           - compiler: "cc"
             os: ubuntu-latest
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: "nBPF"
           - compiler: "clang-14"
             os: ubuntu-22.04
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-memory-sanitizer"
             nBPF: ""
           - compiler: "cc"
             os: macOS-latest
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "" # Disable sanitizer on macos
             nBPF: ""
           - compiler: "cc"
             os: macos-12
             arch: "x86_64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "" # Disable sanitizer on macos
             nBPF: ""
           - compiler: "cc"
             os: ubuntu-latest
             arch: "arm64"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "" # Disable sanitizer on arm64
             nBPF: ""
           - compiler: "cc"
             os: ubuntu-latest
             arch: "armhf"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: "--with-sanitizer"
             nBPF: ""
           - compiler: "cc"
             os: ubuntu-latest
             arch: "s390x"
             gcrypt: ""
-            pcre: "--with-pcre"
+            pcre: "--with-pcre2"
             maxminddb: "--with-maxminddb"
             msan: ""
             nBPF: ""
@@ -209,8 +209,8 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
         run: |
           sudo apt-get install libgcrypt20-dev
-      - name: Install Ubuntu Prerequisites (libpcre)
-        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre')
+      - name: Install Ubuntu Prerequisites (libpcre2)
+        if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2')
         run: |
           sudo apt-get install libpcre3-dev
       - name: Install Ubuntu Prerequisites (maxminddb)
@@ -266,8 +266,8 @@ jobs:
         if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
         run: |
           brew install libgcrypt
-      - name: Install MacOS Prerequisites (libpcre)
-        if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre')
+      - name: Install MacOS Prerequisites (libpcre2)
+        if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2')
         run: |
           brew install pcre
       - name: Install MacOS Prerequisites (maxminddb)

.github/workflows/build_scheduled.yml

@@ -23,7 +23,7 @@ jobs:
           sudo apt-get install libpcre3-dev libmaxminddb-dev lcov
           sudo apt-get install wdiff colordiff
       - name: Configure
-        run: ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre --with-maxminddb --enable-tls-sigs
+        run: ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre2 --with-maxminddb --enable-tls-sigs
       - name: Build
         run: make all
       - name: Test
@@ -91,7 +91,7 @@ jobs:
           pprof -h
       - name: Configure nDPI library
         run: |
-          ./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre --with-maxminddb --enable-tls-sigs
+          ./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs
       - name: Build nDPI library
         run: |
           make

.github/workflows/codeql.yml

@@ -42,7 +42,7 @@ jobs:
           pprof -h
       - name: Configure nDPI library
         run: |
-          ./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre --with-maxminddb --enable-tls-sigs --enable-debug-messages
+          ./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs --enable-debug-messages
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

README.md

@@ -51,7 +51,7 @@ On Windows:
 There are three supported ways to build nDPI:
 
 1. MSYS2 (assuming [MSYS2](https://www.msys2.org/) already installed):
-  - msys2 -c "pacman --noconfirm -S --needed --overwrite '\*' git mingw-w64-x86\_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86\_64-json-c mingw-w64-x86\_64-crt-git mingw-w64-x86\_64-pcre mingw-w64-x86\_64-libpcap"
+  - msys2 -c "pacman --noconfirm -S --needed --overwrite '\*' git mingw-w64-x86\_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86\_64-json-c mingw-w64-x86\_64-crt-git mingw-w64-x86\_64-pcre2 mingw-w64-x86\_64-libpcap"
 
 2. Mingw-w64
 
@@ -79,7 +79,7 @@ The entire procedure of adding new protocols in detail:
 5. Choose (do not change anything) a selection bitmask from: `src/include/ndpi_define.h`
 6. Set protocol default ports in `ndpi_init_protocol_defaults` in: `src/lib/ndpi_main.c`
 7. Be sure to have nBPF support, cloning `PF_RING` in the same directory where you cloned `nDPI`: `git clone https://github.com/ntop/PF_RING/ && cd PF_RING/userland/nbpf && ./configure && make`
-8. From the `nDPI` root directory, `./autogen.sh --with-pcre` (nBPF and PCRE are usually optional, but they are needed to run/update *all* the unit tests)
+8. From the `nDPI` root directory, `./autogen.sh --with-pcre2` (nBPF and PCRE2 are usually optional, but they are needed to run/update *all* the unit tests)
 9. `make`
 10. `make check`
 11. Update the documentation, adding this new protocol to `doc/protocols.rst`

configure.ac

@@ -359,14 +359,14 @@ AS_IF([test "${with_local_libgcrypt+set}" = set],[
   AC_DEFINE_UNQUOTED(USE_HOST_LIBGCRYPT, 1, [Use locally installed libgcrypt instead of builtin gcrypt-light])
 ])
 
-dnl> PCRE
-PCRE_ENABLED=0
-AC_ARG_WITH(pcre, AS_HELP_STRING([--with-pcre], [Enable nDPI build with libpcre]))
-if test "${with_pcre+set}" = set; then :
-  AC_CHECK_LIB(pcre, pcre_compile, AC_DEFINE_UNQUOTED(HAVE_PCRE, 1, [libpcre(-dev) is present]))
-  if test "x$ac_cv_lib_pcre_pcre_compile" = xyes; then :
-    ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lpcre"
-    PCRE_ENABLED=1
+dnl> PCRE2
+PCRE2_ENABLED=0
+AC_ARG_WITH(pcre2, AS_HELP_STRING([--with-pcre2], [Enable nDPI build with libpcre2]))
+if test "${with_pcre2+set}" = set; then :
+  AC_CHECK_LIB(pcre2-8, pcre2_compile_8, AC_DEFINE_UNQUOTED(HAVE_PCRE2, 1, [libpcre2(-dev) is present]))
+  if test "x$ac_cv_lib_pcre2_8_pcre2_compile_8" = xyes; then :
+    ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lpcre2-8"
+    PCRE2_ENABLED=1
   fi
 fi
 
@@ -420,7 +420,7 @@ AC_SUBST(GPROF_CFLAGS)
 AC_SUBST(GPROF_LIBS)
 AC_SUBST(GPROF_ENABLED)
 AC_SUBST(USE_HOST_LIBGCRYPT)
-AC_SUBST(PCRE_ENABLED)
+AC_SUBST(PCRE2_ENABLED)
 AC_SUBST(NBPF_ENABLED)
 AC_SUBST(HANDLE_TLS_SIGS)
 AC_SUBST(DISABLE_NPCAP)

src/lib/ndpi_utils.c

@@ -62,12 +62,12 @@
 
 // #define DEBUG_REASSEMBLY
 
-#ifdef HAVE_PCRE
-#include <pcre.h>
+#ifdef HAVE_PCRE2
+#define PCRE2_CODE_UNIT_WIDTH 8
+#include <pcre2.h>
 
-struct pcre_struct {
-  pcre *compiled;
-  pcre_extra *optimized;
+struct pcre2_struct {
+  pcre2_code *compiled;
 };
 #endif
 
@@ -1702,18 +1702,19 @@ static int ndpi_is_xss_injection(char* query) {
 
 /* ********************************** */
 
-#ifdef HAVE_PCRE
+#ifdef HAVE_PCRE2
 
 static void ndpi_compile_rce_regex() {
-  const char *pcreErrorStr = NULL;
-  int pcreErrorOffset;
+  PCRE2_UCHAR pcreErrorStr[128];
+  PCRE2_SIZE pcreErrorOffset;
+  int pcreErrorCode;
 
   for(int i = 0; i < N_RCE_REGEX; i++) {
-    comp_rx[i] = (struct pcre_struct*)ndpi_malloc(sizeof(struct pcre_struct));
+    comp_rx[i] = (struct pcre2_struct*)ndpi_malloc(sizeof(struct pcre2_struct));
 
-    comp_rx[i]->compiled = pcre_compile(rce_regex[i], 0, &pcreErrorStr,
+    comp_rx[i]->compiled = pcre2_compile((PCRE2_SPTR)rce_regex[i], PCRE2_ZERO_TERMINATED, 0, &pcreErrorCode,
                                         &pcreErrorOffset, NULL);
-
+    pcre2_get_error_message(pcreErrorCode, pcreErrorStr, 128);
     if(comp_rx[i]->compiled == NULL) {
 #ifdef DEBUG
       NDPI_LOG_ERR(ndpi_str, "ERROR: Could not compile '%s': %s\n", rce_regex[i],
@@ -1723,18 +1724,16 @@ static void ndpi_compile_rce_regex() {
       continue;
     }
 
-    comp_rx[i]->optimized = pcre_study(comp_rx[i]->compiled, 0, &pcreErrorStr);
+    pcreErrorCode = pcre2_jit_compile(comp_rx[i]->compiled, PCRE2_JIT_COMPLETE);
 
 #ifdef DEBUG
-    if(pcreErrorStr != NULL) {
-      NDPI_LOG_ERR(ndpi_str, "ERROR: Could not study '%s': %s\n", rce_regex[i],
+    if(pcreErrorCode < 0) {
+      pcre2_get_error_message(pcreErrorCode, pcreErrorStr, 128);
+      NDPI_LOG_ERR(ndpi_str, "ERROR: Could not jit compile '%s': %s\n", rce_regex[i],
                    pcreErrorStr);
     }
 #endif
   }
-
-  if(pcreErrorStr != NULL)
-    ndpi_free((void *)pcreErrorStr);
 }
 
 /* ********************************** */
@@ -1745,17 +1744,17 @@ static int ndpi_is_rce_injection(char* query) {
     initialized_comp_rx = 1;
   }
 
+  pcre2_match_data *pcreMatchData;
   int pcreExecRet;
-  int subStrVec[30];
 
   for(int i = 0; i < N_RCE_REGEX; i++) {
     unsigned int length = strlen(query);
 
-    pcreExecRet = pcre_exec(comp_rx[i]->compiled,
-                            comp_rx[i]->optimized,
-                            query, length, 0, 0, subStrVec, 30);
-
-    if(pcreExecRet >= 0) {
+    pcreMatchData = pcre2_match_data_create_from_pattern(comp_rx[i]->compiled, NULL);
+    pcreExecRet = pcre2_match(comp_rx[i]->compiled,
+                            (PCRE2_SPTR)query, length, 0, 0, pcreMatchData, NULL);
+    pcre2_match_data_free(pcreMatchData);
+    if(pcreExecRet > 0) {
       return 1;
     }
 #ifdef DEBUG
@@ -1845,7 +1844,7 @@ ndpi_risk_enum ndpi_validate_url(char *url) {
 	    rc = NDPI_URL_POSSIBLE_XSS;
 	  else if(ndpi_is_sql_injection(decoded))
 	    rc = NDPI_URL_POSSIBLE_SQL_INJECTION;
-#ifdef HAVE_PCRE
+#ifdef HAVE_PCRE2
 	  else if(ndpi_is_rce_injection(decoded))
 	    rc = NDPI_URL_POSSIBLE_RCE_INJECTION;
 #endif

src/lib/third_party/include/rce_injection.h

@@ -1,4 +1,4 @@
-#ifdef HAVE_PCRE
+#ifdef HAVE_PCRE2
 
 #ifndef NDPI_RCE_H
 #define NDPI_RCE_H
@@ -8,7 +8,7 @@
 #define N_RCE_REGEX 7
 
 /* Compiled regex */
-static struct pcre_struct *comp_rx[N_RCE_REGEX];
+static struct pcre2_struct *comp_rx[N_RCE_REGEX];
 
 static unsigned int initialized_comp_rx = 0;
 
@@ -615,4 +615,4 @@ static const char *pwsh_commands[] = {
   "-PSConsoleFile"
 };
 
-#endif //HAVE_PCRE
+#endif //HAVE_PCRE2

tests/do.sh.in

@@ -26,7 +26,7 @@ CMD_COLORDIFF="$(which colordiff)"
 
 EXE_SUFFIX=@EXE_SUFFIX@
 GPROF_ENABLED=@GPROF_ENABLED@
-PCRE_ENABLED=@PCRE_ENABLED@
+PCRE2_ENABLED=@PCRE2_ENABLED@
 PCRE_PCAPS="WebattackRCE.pcap"
 NBPF_ENABLED=@NBPF_ENABLED@
 NBPF_PCAPS="h323-overflow.pcap"
@@ -84,7 +84,7 @@ check_results() {
 		[ $SKIP_PCAP = 1 ] && continue
 	    fi
 	    SKIP_PCAP=0
-	    if [ $PCRE_ENABLED -eq 0 ]; then
+	    if [ $PCRE2_ENABLED -eq 0 ]; then
 	      for p in $PCRE_PCAPS; do
 	        if [ $f = $p ]; then
 	          SKIP_PCAP=1