Merge pull request #74 from dns-stats/develop

Merge develop into master for 1.2.1-beta2
dns-stats · Sep 23, 2021 · 31d4030 · 31d4030
2 parents 7000b99 + 995b3f0
commit 31d4030
Show file tree

Hide file tree

Showing 35 changed files with 405 additions and 97 deletions.
diff --git a/ChangeLog.txt b/ChangeLog.txt
@@ -6,6 +6,19 @@
 # file, you can obtain one at https://mozilla.org/MPL/2.0/.
 #
 
+Version 1.2.1-beta2 -
+--------------------------
+* Bump the private version because schema extended with new stats.
+* Add new stats to C-DNS: sniffer drops, matcher drops and sampling discards,
+  and 3 libpcap stats. The libpcap stats should not be considered reliable and 
+  provided for information only. 
+* Fix file durations for files with more than one C-DNS block in them so high
+  traffic rates can be correctly calculated from the statistics.
+* Constrain size of matcher queue to reduce memory consumption when 
+  dropping/sampling.
+* Add `-D` option (--relaxed-mode). With this option a warning (not an error) is 
+  issued for unrecognized command line and config file options. 
+
 Version 1.2.1-beta1 - 2021-08-23
 --------------------------
 

diff --git a/doc/c-dns.cddl b/doc/c-dns.cddl
@@ -20,7 +20,7 @@ File = [
 FilePreamble = {
     major-format-version => 1,
     minor-format-version => 0,
-    ? private-version    => 2,       ; Compactor private indicator
+    ? private-version    => 3,       ; Compactor private indicator
     block-parameters     => [+ BlockParameters],
 }
 major-format-version = 0
@@ -236,6 +236,12 @@ BlockStatistics = {
     ? compactor-missing-packets      => uint,
     ? compactor-missing-non-dns      => uint,
     ? compactor-packets              => uint,
+    ? compactor-missing-received     => uint,
+    ? compactor-discarded-packets    => uint,
+    ? compactor-missing-matcher      => uint,
+    ? pcap-packets                   => uint,
+    ? pcap-missing-if                => uint,
+    ? pcap-missing-os                => uint,
 }
 processed-messages  = 0
 qr-data-items       = 1
@@ -250,6 +256,12 @@ compactor-missing-pairs          = -3
 compactor-missing-packets        = -4
 compactor-missing-non-dns        = -5
 compactor-packets                = -6
+compactor-missing-received       = -7
+compactor-discarded-packets      = -8
+compactor-missing-matcher        = -9
+pcap-packets                     = -10
+pcap-missing-if                  = -11
+pcap-missing-os                  = -12
 
 ;
 ; Tables of common data referenced from records in a Block.

diff --git a/doc/inspector.adoc.in b/doc/inspector.adoc.in
@@ -85,6 +85,9 @@ output file must be specified with the *--output* option.
 *-r. --report-info*::
   Report info (config and statistics summary) to standard output on exit.
 
+*-D. --relaxed-mode*::
+  Warn (instead of error) if unrecognized command line options are found.
+
 *-N, --no-output*::
   Don't write any output PCAP or template files, just generate any requested ancillary
   outputs e.g. `.info` files.

diff --git a/doc/user-guide/compactor-command-options.adoc.in b/doc/user-guide/compactor-command-options.adoc.in
@@ -8,11 +8,16 @@ Command options are given on the command line.
 *-v, --version*::
   Print the version number of *compactor* to the standard output stream and then exit.
 
-*-r. --report-info* [_arg_]::
+*-r, --report-info* [_arg_]::
   Report info (config and statistics summary) on exit. _arg_ may be
   `true` or `1` to enable promiscuous mode, `false` or `0` to disable
   promiscuous mode. If _arg_ is omitted, it defaults to `true`.
 
+*-D, --relaxed-mode* [_arg_]::
+  Warn (instead of error) if unrecognized command line or config file options are found.
+  _arg_ may be `true` or `1` to enable promiscuous mode, `false` or `0` to disable
+   promiscuous mode. If _arg_ is omitted, it defaults to `true`.
+
 *-l, --list-interfaces*::
   List all network interfaces from which DNS traffic may be captured.
 

diff --git a/doc/user-guide/overview.adoc.in b/doc/user-guide/overview.adoc.in
@@ -95,7 +95,7 @@ _compactor_ 1.0 and later writes C-DNS as described in
 https://tools.ietf.org/html/rfc8618[RFC8618],
 with the following changes:
 
-* A private version ID of 2 is present.
+* A private version ID of 3 is present.
 * Some _compactor_ implementation-specific entries are added
   to several C-DNS maps (see RFC section 7.1). They are listed below
   with their _compactor_ key values in parenthesis.
@@ -123,6 +123,15 @@ with the following changes:
 *** _compactor-missing-non-dns_ (-5): count of output ignored PCAP packets not written
     by _compactor_ because they could not be processed quickly enough.
 *** _compactor-packets_ (-6): total packets received by compactor
+*** _compactor-missing-received_ (-7): count of packets sniffed from the network not processed
+    by _compactor_ because they could not be processed quickly enough.
+*** _compactor-discarded-packets_ (-8): count of packets actively discarded by compactor due
+    to some processing threshold, e.g. sampling.
+*** _compactor-missing_matcher_ (-9): count of packets unmatched packets not written
+    by _compactor_ because they could not be processed for matching quickly enough.
+*** _pcap-packets_ (-10): informational only report from pcap library - count of packets received
+*** _pcap-missing-if_ (-11): informational only report from pcap library - count of packets dropped at the interface
+*** _pcap-missing-os_ (-12): informational only report from pcap library - count of packets dropped in the kernel
 
 [IMPORTANT]
 ====

diff --git a/doc/user-guide/running.adoc.in b/doc/user-guide/running.adoc.in
@@ -44,15 +44,17 @@ Usage: compactor  [options] [capture-file ...]
 Options:
 
 Command options:
-  -h [ --help ]                   show this help message.
-  -v [ --version ]                show version information.
-  -c [ --configfile ] arg         configuration file.
-  --excludesfile arg              exclude hints file.
-  -r [ --report-info ] [=arg(=1)] report info (config and stats summary) on 
-                                  exit.
-  --debug-dns [=arg(=1)]          print DNS packet details.
-  --debug-qr [=arg(=1)]           print Query/Response match details.
-  -l [ --list-interfaces ]        list all network interfaces.
+  -h [ --help ]                    show this help message.
+  -v [ --version ]                 show version information.
+  -c [ --configfile ] arg          configuration file.
+  --excludesfile arg               exclude hints file.
+  -r [ --report-info ] [=arg(=1)]  report info (config and stats summary) on 
+                                   exit.
+  -D [ --relaxed-mode ] [=arg(=1)] parse command line allowing 
+                                   unrecognized options but warning.
+  --debug-dns [=arg(=1)]           print DNS packet details.
+  --debug-qr [=arg(=1)]            print Query/Response match details.
+  -l [ --list-interfaces ]         list all network interfaces.
 
 Configuration:
   -t [ --rotation-period ] arg (=300)   rotation period for all outputs, in 
@@ -297,20 +299,20 @@ messages.
 | Comment
 
 | ERROR
-| Dropping on these channels: Sniffer C-DNS
+| Dropping on these channels: Sniffer Matcher C-DNS 
 | Packets are arriving faster than _compactor_  can
 process them, so packets are being dropped internally on the specified
 channels. More information is available via the *--log-network-stats-period* logging output.
 
 | ERROR
-| Ignored PCAP channel overflow. Dropping packet(s).
+| Dropping on these channels: Ignored-PCAP
 | More ignored packets - that is, packets that do not appear to be DNS
 related or which are malformed - are arriving than can be processed
 and recorded to the ignored PCAP output. At least one has been
 dropped.
 
 | ERROR
-| Raw PCAP channel overflow. Dropping packet(s).
+| Dropping on these channels: Raw-PCAP
 | More input packets are arriving than can be processed and recorded
 to the raw PCAP output. At least one has been dropped.
 
@@ -348,18 +350,31 @@ time limit.
   stopping collection.
 |===
 
+Other error messages are reporting an internal error.
+
 Example statistics produced by enabling the *log-network-stats-period* option:
 
 ----
-*Stats interval: average rate         480027 pps  over  1s
- LIBPCAP : recv/OS drop/IF drop       481002/         0/         0
- Sniffer : recv/dropped/queue         480701/         0/       917
- Sampling: recv/discard               480027/         0
- C-DNS   : recv/dropped/queue         480027/         0/         0
+*Stats interval: average rate           1896 pps  over  1s
+ LIBPCAP : recv/OS drop/IF drop         1896/         0/         0
+ Sniffer : recv/dropped/queue           1896/         0/         1
+ Matcher : recv/dropped/queue           1896/         0/         0
+ CDNS    : recv/dropped/queue           1896/         0/         0
+ CDNS out: writ/% traffic               1896/       100/          
  PCAP out: raw drop/ignored drop           0/         0/
+
 ----
 
-Other error messages are reporting an internal error.
+These statistics provide detail about the internal components of compactor that
+may need to drop packets under heavy load. If sampling is enabled, an additional 
+line outputs data on sampling:
+
+----
+ Sampling: recv/discard/state            1896/         0/       OFF
+----
+
+Note that the LIBPCAP statistics provided here are information only and may not be
+reliable, particularly at high load.
 
 === _compactor_  performance considerations
 
@@ -478,6 +493,8 @@ Options:
   -q [ --query-only ]                   write only query messages to output.
   -r [ --report-info ]                  report info (config and stats summary)
                                         on exit.
+  -D [ --relaxed-mode ]                 parse command line allowing 
+                                        unrecognized options but warning.
   -N [ --no-output ]                    do not output PCAP or template files,
                                         only ancillary files, e.g. info files,
                                         for each input.
@@ -899,7 +916,11 @@ Then follows some overall statistics on the capture.
 
 ----
 STATISTICS:
+  Total Packets received                   : 17493
+  Dropped packets at sniffer    (overload) : 0
   Total Packets processed                  : 17493
+  Dropped Matcher messages      (overload) : 0
+  Discarded C-DNS messages      (sampling) : 0
   Processed DNS messages           (C-DNS) : 16529
   Matched DNS query/response pairs (C-DNS) : 8263
   Unmatched DNS queries            (C-DNS) : 2
@@ -908,14 +929,20 @@ STATISTICS:
   Malformed DNS messages           (C-DNS) : 2
   Non-DNS packets                          : 161
   Out-of-order DNS query/responses         : 0
-  Dropped C-DNS items (overload)           : 0
-  Dropped raw PCAP packets (overload)      : 0
-  Dropped non-DNS packets (overload)       : 0
+  Dropped raw PCAP packets      (overload) : 0
+  Dropped non-DNS packets       (overload) : 0
+
+PCAP STATISTICS:
+  Packets received               (libpcap) : 17493
+  Packets dropped at i/f         (libpcap) : 0
+  Packets dropped in kernel      (libpcap) : 0
 ----
 
 NOTE: The above counts are based on traffic as reported to _compactor_ by the
 underlying _libpcap_ library. When under load, this may not reflect actual traffic.
 
+When processing from capture files, the `PCAP STATISTICS` will all be 0.
+
 And finally counts of occurrences of various events recorded, and associated
 addresses.
 

diff --git a/src/blockcbor.cpp b/src/blockcbor.cpp
@@ -35,7 +35,7 @@ namespace block_cbor {
     /**
      * \brief Current output format private version.
      */
-    const unsigned FILE_FORMAT_10_PRIVATE_VERSION = 2;
+    const unsigned FILE_FORMAT_10_PRIVATE_VERSION = 3;
 
     /**
      * \brief Current output format major version.

diff --git a/src/blockcbor.hpp b/src/blockcbor.hpp
@@ -470,6 +470,12 @@ namespace block_cbor {
         compactor_missing_packets,
         compactor_missing_non_dns,
         compactor_packets,
+        compactor_missing_received,
+        compactor_discarded_packets,
+        compactor_missing_matcher,
+        pcap_packets,
+        pcap_missing_if,
+        pcap_missing_os,
 
         // Obsolete
         partially_malformed_packets,
@@ -981,6 +987,12 @@ namespace block_cbor {
         BlockStatisticsField::compactor_missing_packets,
         BlockStatisticsField::compactor_missing_non_dns,
         BlockStatisticsField::compactor_packets,
+        BlockStatisticsField::compactor_missing_received,
+        BlockStatisticsField::compactor_discarded_packets,
+        BlockStatisticsField::compactor_missing_matcher,
+        BlockStatisticsField::pcap_packets,
+        BlockStatisticsField::pcap_missing_if,
+        BlockStatisticsField::pcap_missing_os,
     };
 
     /**

diff --git a/src/blockcbordata.cpp b/src/blockcbordata.cpp
@@ -1627,6 +1627,30 @@ namespace block_cbor {
                 last_packet_statistics.raw_packet_count += dec.read_unsigned();
                 break;
 
+            case BlockStatisticsField::compactor_missing_received:
+                last_packet_statistics.sniffer_drop_count += dec.read_unsigned();
+                break;
+
+            case BlockStatisticsField::compactor_discarded_packets:
+                last_packet_statistics.discarded_sampling_count += dec.read_unsigned();
+                break;
+
+            case BlockStatisticsField::compactor_missing_matcher:
+                last_packet_statistics.matcher_drop_count += dec.read_unsigned();
+                break;
+
+            case BlockStatisticsField::pcap_packets:
+                last_packet_statistics.pcap_recv_count += dec.read_unsigned();
+                break;
+
+            case BlockStatisticsField::pcap_missing_if:
+                last_packet_statistics.pcap_ifdrop_count += dec.read_unsigned();
+                break;
+
+            case BlockStatisticsField::pcap_missing_os:
+                last_packet_statistics.pcap_drop_count += dec.read_unsigned();
+                break;
+
             default:
                 dec.skip();
                 break;
@@ -1816,6 +1840,12 @@ namespace block_cbor {
         constexpr int missing_packets_index = find_block_statistics_index(BlockStatisticsField::compactor_missing_packets);
         constexpr int missing_non_dns_index = find_block_statistics_index(BlockStatisticsField::compactor_missing_non_dns);
         constexpr int packets_index = find_block_statistics_index(BlockStatisticsField::compactor_packets);
+        constexpr int missing_received_index = find_block_statistics_index(BlockStatisticsField::compactor_missing_received);
+        constexpr int discarded_packets_index = find_block_statistics_index(BlockStatisticsField::compactor_discarded_packets);
+        constexpr int missing_matcher_index = find_block_statistics_index(BlockStatisticsField::compactor_missing_matcher);
+        constexpr int pcap_packets_index = find_block_statistics_index(BlockStatisticsField::pcap_packets);
+        constexpr int pcap_missing_if_index = find_block_statistics_index(BlockStatisticsField::pcap_missing_if);
+        constexpr int pcap_missing_os_index = find_block_statistics_index(BlockStatisticsField::pcap_missing_os);
 
         enc.writeMapHeader();
         enc.write(processed_messages_index);
@@ -1842,6 +1872,18 @@ namespace block_cbor {
         enc.write(last_packet_statistics.output_ignored_pcap_drop_count - start_packet_statistics.output_ignored_pcap_drop_count);
         enc.write(packets_index);
         enc.write(last_packet_statistics.raw_packet_count - start_packet_statistics.raw_packet_count);
+        enc.write(missing_received_index);
+        enc.write(last_packet_statistics.sniffer_drop_count - start_packet_statistics.sniffer_drop_count);
+        enc.write(discarded_packets_index);
+        enc.write(last_packet_statistics.discarded_sampling_count - start_packet_statistics.discarded_sampling_count);
+        enc.write(missing_matcher_index);
+        enc.write(last_packet_statistics.matcher_drop_count - start_packet_statistics.matcher_drop_count);
+        enc.write(pcap_packets_index);
+        enc.write(last_packet_statistics.pcap_recv_count - start_packet_statistics.pcap_recv_count);
+        enc.write(pcap_missing_if_index);
+        enc.write(last_packet_statistics.pcap_ifdrop_count - start_packet_statistics.pcap_ifdrop_count);
+        enc.write(pcap_missing_os_index);
+        enc.write(last_packet_statistics.pcap_drop_count - start_packet_statistics.pcap_drop_count);
         enc.writeBreak();
     }
 

diff --git a/src/blockcborwriter.cpp b/src/blockcborwriter.cpp
@@ -116,6 +116,7 @@ void BlockCborWriter::startRecord(const std::shared_ptr<QueryResponse>& qr)
         const DNSMessage &d(qr->has_query() ? qr->query() : qr->response());
         data_->end_time = d.timestamp;
         writeBlock();
+        data_->start_time = d.timestamp;
     }
     query_response_.clear();
     clear_in_progress_extra_info();