Added generic user agent setter. (ntop#1530)

* ndpiReader: Print user agent if one was set and not just for certain protocols.

Signed-off-by: lns <[email protected]>

example/reader_util.c

@@ -1116,11 +1116,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
       flow->http.response_status_code = flow->ndpi_flow->http.response_status_code;
       ndpi_snprintf(flow->http.content_type, sizeof(flow->http.content_type), "%s", flow->ndpi_flow->http.content_type ? flow->ndpi_flow->http.content_type : "");
       ndpi_snprintf(flow->http.request_content_type, sizeof(flow->http.request_content_type), "%s", flow->ndpi_flow->http.request_content_type ? flow->ndpi_flow->http.request_content_type : "");
-      ndpi_snprintf(flow->http.user_agent, sizeof(flow->http.user_agent), "%s", flow->ndpi_flow->http.user_agent ? flow->ndpi_flow->http.user_agent : "");
     }
-  } else if(is_ndpi_proto(flow, NDPI_PROTOCOL_SSDP) ||
-            is_ndpi_proto(flow, NDPI_PROTOCOL_XIAOMI)) {
-    ndpi_snprintf(flow->http.user_agent, sizeof(flow->http.user_agent), "%s", flow->ndpi_flow->http.user_agent ? flow->ndpi_flow->http.user_agent : "");
   } else if(is_ndpi_proto(flow, NDPI_PROTOCOL_TELNET)) {
     if(flow->ndpi_flow->protos.telnet.username[0] != '\0')
       flow->telnet.username = ndpi_strdup(flow->ndpi_flow->protos.telnet.username);
@@ -1145,8 +1141,6 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
 	  ) {
     flow->ssh_tls.ssl_version = flow->ndpi_flow->protos.tls_quic.ssl_version;
 
-    ndpi_snprintf(flow->http.user_agent, sizeof(flow->http.user_agent), "%s", flow->ndpi_flow->http.user_agent ? flow->ndpi_flow->http.user_agent : "");
-
     if(flow->ndpi_flow->protos.tls_quic.server_names_len > 0 && flow->ndpi_flow->protos.tls_quic.server_names)
       flow->ssh_tls.server_names = ndpi_strdup(flow->ndpi_flow->protos.tls_quic.server_names);
 
@@ -1224,6 +1218,10 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
     }
   }
 
+  ndpi_snprintf(flow->http.user_agent,
+                sizeof(flow->http.user_agent),
+                "%s", (flow->ndpi_flow->http.user_agent ? flow->ndpi_flow->http.user_agent : ""));
+
   if(flow->detection_completed && (!flow->check_extra_packets)) {
     if(is_ndpi_proto(flow, NDPI_PROTOCOL_UNKNOWN)) {
       if(workflow->__flow_giveup_callback != NULL)

src/include/ndpi_main.h

@@ -161,6 +161,7 @@ extern "C" {
 			    const char *alpn_to_check, u_int alpn_to_check_len);    
 
   char *ndpi_hostname_sni_set(struct ndpi_flow_struct *flow, const u_int8_t *value, size_t value_len);
+  char *ndpi_user_agent_set(struct ndpi_flow_struct *flow, const u_int8_t *value, size_t value_len);
 
 #ifdef __cplusplus
 }

src/lib/ndpi_main.c

@@ -8421,3 +8421,22 @@ char *ndpi_hostname_sni_set(struct ndpi_flow_struct *flow, const u_int8_t *value
 
   return dst;
 }
+
+/* ******************************************************************** */
+
+char *ndpi_user_agent_set(struct ndpi_flow_struct *flow, const u_int8_t *value, size_t value_len)
+{
+  if (flow->http.user_agent != NULL)
+  {
+    return NULL;
+  }
+
+  flow->http.user_agent = ndpi_malloc(value_len + 1);
+  if (flow->http.user_agent != NULL)
+  {
+    memcpy(flow->http.user_agent, value, value_len);
+    flow->http.user_agent[value_len] = '\0';
+  }
+
+  return flow->http.user_agent;
+}

src/lib/protocols/http.c

@@ -535,14 +535,11 @@ int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
     }
   }
 
-  if(flow->http.user_agent == NULL) {
-    flow->http.user_agent = ndpi_malloc(ua_ptr_len + 1);
-    if(flow->http.user_agent) {
-      memcpy(flow->http.user_agent, (char*)ua_ptr, ua_ptr_len);
-      flow->http.user_agent[ua_ptr_len] = '\0';
-
-      ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent);
-    }
+  if (ndpi_user_agent_set(flow, ua_ptr, ua_ptr_len) != NULL)
+  {
+    ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent);
+  } else {
+    NDPI_LOG_DBG2(ndpi_struct, "Could not set HTTP user agent\n");
   }
 
   NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n",

src/lib/protocols/ssdp.c

@@ -38,13 +38,9 @@ static void ssdp_parse_lines(struct ndpi_detection_module_struct
 
   /* Save user-agent for device discovery if available */
   if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) {
-    if(flow->http.user_agent == NULL) {
-      flow->http.user_agent = ndpi_malloc(packet->user_agent_line.len + 1);
-      if(flow->http.user_agent) {
-        memcpy(flow->http.user_agent,
-          (char*)packet->user_agent_line.ptr, packet->user_agent_line.len);
-        flow->http.user_agent[packet->user_agent_line.len] = '\0';
-      }
+    if (ndpi_user_agent_set(flow, packet->user_agent_line.ptr, packet->user_agent_line.len) == NULL)
+    {
+      NDPI_LOG_DBG2(ndpi_struct, "Could not set SSDP user agent\n");
     }
   }
 }

src/lib/protocols/xiaomi.c

@@ -52,10 +52,9 @@ static void xiaomi_dissect_metadata(struct ndpi_detection_module_struct *ndpi_st
 
     switch(op) {
       case 0x12:
-        flow->http.user_agent = ndpi_malloc(len + 1);
-        if(flow->http.user_agent != NULL) {
-          memcpy(flow->http.user_agent, &payload[offset], len);
-          flow->http.user_agent[len] = '\0';
+        if (ndpi_user_agent_set(flow, &payload[offset], len) == NULL)
+        {
+            NDPI_LOG_DBG2(ndpi_struct, "Could not set Xiaomi user agent\n");
         }
         break;
 

tests/result/KakaoTalk_talk.pcap.out

@@ -29,7 +29,7 @@ JA3 Host Stats:
 	5	TCP 10.24.82.188:59954 <-> 173.252.88.128:443 [proto: 91.119/TLS.Facebook][Encrypted][Confidence: DPI][cat: SocialNetwork/6][15 pkts/2932 bytes <-> 14 pkts/1092 bytes][Goodput ratio: 71/27][1.96 sec][bytes ratio: 0.457 (Upload)][IAT c2s/s2c min/avg/max/stddev: 2/0 141/117 494/295 163/92][Pkt Len c2s/s2c min/avg/max/stddev: 56/56 195/78 735/189 228/35][Risk: ** Obsolete TLS Version (1.1 or older) **** Possibly Malicious JA3 Fingerprint **][Risk Score: 150][TLSv1][JA3C: dff8a0aa1c904aaea76c5bf624e88333][JA3S: 07dddc59e60135c7b479d39c3ae686af][Cipher: TLS_ECDHE_ECDSA_WITH_RC4_128_SHA][Plen Bins: 30,23,0,0,15,0,7,0,7,0,0,0,0,0,0,0,0,0,0,0,0,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	6	UDP 10.24.82.188:10269 <-> 1.201.1.174:23047 [proto: 194/KakaoTalk_Voice][ClearText][Confidence: DPI][cat: VoIP/10][12 pkts/1692 bytes <-> 10 pkts/1420 bytes][Goodput ratio: 69/69][45.10 sec][bytes ratio: 0.087 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 1062/3176 4203/4247 4716/5160 1131/719][Pkt Len c2s/s2c min/avg/max/stddev: 122/142 141/142 150/142 6/0][Plen Bins: 0,0,4,95,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	7	UDP 10.24.82.188:11321 <-> 1.201.1.174:23045 [proto: 194/KakaoTalk_Voice][ClearText][Confidence: DPI][cat: VoIP/10][11 pkts/1542 bytes <-> 11 pkts/1542 bytes][Goodput ratio: 69/69][43.84 sec][bytes ratio: 0.000 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 1105/1052 4266/3766 4903/4991 1245/1144][Pkt Len c2s/s2c min/avg/max/stddev: 122/122 140/140 142/142 6/6][Plen Bins: 0,0,9,90,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-	8	TCP 10.24.82.188:48489 <-> 203.205.147.215:80 [proto: 285.48/Tencent.QQ][ClearText][Confidence: DPI][cat: Chat/9][8 pkts/1117 bytes <-> 7 pkts/610 bytes][Goodput ratio: 54/34][3.79 sec][Hostname/SNI: hkminorshort.weixin.qq.com][bytes ratio: 0.294 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/51 406/439 2019/1166 732/515][Pkt Len c2s/s2c min/avg/max/stddev: 56/56 140/87 665/262 199/71][Risk: ** Known Protocol on Non Standard Port **][Risk Score: 50][PLAIN TEXT (POST http)][Plen Bins: 0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+	8	TCP 10.24.82.188:48489 <-> 203.205.147.215:80 [proto: 285.48/Tencent.QQ][ClearText][Confidence: DPI][cat: Chat/9][8 pkts/1117 bytes <-> 7 pkts/610 bytes][Goodput ratio: 54/34][3.79 sec][Hostname/SNI: hkminorshort.weixin.qq.com][bytes ratio: 0.294 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/51 406/439 2019/1166 732/515][Pkt Len c2s/s2c min/avg/max/stddev: 56/56 140/87 665/262 199/71][User-Agent: MicroMessenger Client][Risk: ** Known Protocol on Non Standard Port **][Risk Score: 50][PLAIN TEXT (POST http)][Plen Bins: 0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	9	TCP 10.24.82.188:51021 <-> 103.246.57.251:8080 [proto: 131/HTTP_Proxy][ClearText][Confidence: Match by port][cat: Web/5][6 pkts/543 bytes <-> 5 pkts/945 bytes][Goodput ratio: 25/64][24.77 sec][bytes ratio: -0.270 (Download)][IAT c2s/s2c min/avg/max/stddev: 77/47 4920/8061 17431/17434 6679/7163][Pkt Len c2s/s2c min/avg/max/stddev: 68/68 90/189 130/504 24/164][Plen Bins: 16,51,0,16,0,0,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	10	TCP 139.150.0.125:443 <-> 10.24.82.188:46947 [proto: 91/TLS][Encrypted][Confidence: Match by port][cat: Web/5][3 pkts/1044 bytes <-> 2 pkts/154 bytes][Goodput ratio: 84/27][51.90 sec][Plen Bins: 0,33,0,0,0,0,0,0,0,0,0,0,0,66,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
 	11	TCP 10.24.82.188:58916 <-> 54.255.185.236:5222 [proto: 265/AmazonAWS][Encrypted][Confidence: Match by IP][cat: Cloud/13][2 pkts/225 bytes <-> 2 pkts/171 bytes][Goodput ratio: 39/20][0.46 sec][PLAIN TEXT (xiaomi.com)][Plen Bins: 0,50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]