diff --git a/ChangeLog.txt b/ChangeLog.txt index 7659492..2e4436c 100644 --- a/ChangeLog.txt +++ b/ChangeLog.txt @@ -6,6 +6,13 @@ # file, you can obtain one at https://mozilla.org/MPL/2.0/. # +Version 1.2.2-rc1 - 2022-05-24 +-------------------------- + +* Extend SIGUSR1 mechanism to DNSTAP capture +* Align DNSTAP stats logging better with network capture stats logging +* Various minor updates to documentation + Version 1.2.2-beta1 - 2022-05-18 -------------------------- diff --git a/INSTALL b/INSTALL index 8866325..4516d2d 100644 --- a/INSTALL +++ b/INSTALL @@ -8,7 +8,7 @@ https://github.com/dns-stats/compactor/wiki/user-guide.pdf. = Installing from packages -Binary install packages for Ubuntu 16.04 'Xenial Xerus' are available +Binary install packages for Ubuntu are available from a Launchpad Personal Package Archive (PPA). You need first to add the DNS-STATS PPA to your system's Software Sources: @@ -49,8 +49,11 @@ Optional tools: Required libraries: * The following Boost (http://www.boost.org) libraries: + - boost-filesystem + - boost-iostreams - boost-log - boost-program-options + - boost-system - boost-thread - boost-iostreams * liblzma (http://tukaani.org/xz/) diff --git a/KNOWN_ISSUES.txt b/KNOWN_ISSUES.txt index e2fc859..832a000 100644 --- a/KNOWN_ISSUES.txt +++ b/KNOWN_ISSUES.txt @@ -22,10 +22,15 @@ KNOWN ISSUES/LIMITATIONS: packets. This means that under low traffic conditions files may not rotate until the first packet arrives after the rotation period has expired, and so files may be much longer than the file rotation period in this case. File rotation - may be forced externally by sending a SIGUSR1 to the compactor. It is recommended - to use either the configuration parameters OR an external signal to trigger - file rotation (to avoid a race condition). The file rotation mechanism is - under review and may be further updated in a coming release. + may be forced externally by sending a SIGUSR1 to the compactor. It is + recommended to either + * rely on just the file rotation configuration parameters OR + * to override the internally triggered rotation with an external signal sent + at a period much shorter then the file-rotation-period (which defaults to 300s) + The second mechanism must be used with care to avoid a race condition because + the internal triggering cannot currently be disabled if the filename contains + a timestamp. The file rotation mechanism is under review and may be further + updated in a coming release. * File rotation period reported by the inspector is calculated as the difference in seconds between the first and last items read from the @@ -44,7 +49,7 @@ KNOWN ISSUES/LIMITATIONS: captured. This is a problem with the libpcap library and the kernel. Debian Jessie can be used by updating the kernel to a later backported kernel; we have tested this with the 4.7.0 kernel. For Ubuntu we - recommend upgrading to Xenial. + recommend upgrading to at least Xenial. * The design of C-DNS allows blocks with different storage and collection parameters to be present in the same C-DNS file. diff --git a/doc/inspector.adoc.in b/doc/inspector.adoc.in index 9baca4d..069ced5 100644 --- a/doc/inspector.adoc.in +++ b/doc/inspector.adoc.in @@ -370,6 +370,9 @@ e.g. `{{client_address:x-ipaddr}}`. | `x-date` | Output timestamp in seconds as a UTC date in ISO601 format (YYYY-MM-DD). +| `x-datefmt=` +| Output timestamp in seconds as a UTC date and time in user-specified format. The string `` must be a valid *strftime(3)* format string. + | `x-datetime` | Output timestamp in seconds as a UTC date and time in ISO601 format (YYYY-MM-DD HH:MM:SS). diff --git a/doc/user-guide/configuring.adoc.in b/doc/user-guide/configuring.adoc.in index e573064..ec37e89 100644 --- a/doc/user-guide/configuring.adoc.in +++ b/doc/user-guide/configuring.adoc.in @@ -377,11 +377,11 @@ compactor's use of memory and CPU. ==== Linux with `systemd` By default, http://releases.ubuntu.com/16.04/[Ubuntu 16.04 LTS 'Xenial -Xerus'] uses `systemd`. +Xerus'] and later releases use `systemd`. ===== Running as a daemon -Binary packages for Ubuntu 16.04 include a `systemd` service file with +Binary packages for Ubuntu include a `systemd` service file with the setup information required to run _compactor_ as a daemon. When installing on Debian-based systems such as Ubuntu, installing the package diff --git a/doc/user-guide/installation.adoc.in b/doc/user-guide/installation.adoc.in index 906463e..fc8bc99 100644 --- a/doc/user-guide/installation.adoc.in +++ b/doc/user-guide/installation.adoc.in @@ -21,8 +21,7 @@ _compactor_ and _inspector_ are supplied in separate packages named Both _compactor_ and _inspector_ use https://libtins.github.io/[libtins] for various network related -functions. There is no suitable pre-packaged version for Ubuntu Xenial, so -a package is supplied along with the DNS-STATS packages. It will be installed +functions. It will be installed as a pre-requisite package automatically. ===== Installing @@ -125,6 +124,7 @@ http://www.openssl.org/. | `libctemplate` | Text templating system. https://github.com/OlafvdSpek/ctemplate. | `libmaxminddb` | MaxMind GeoIP reader. https://github.com/maxmind/libmaxminddb. +| `protobuf` | Google Protocol Buffers library. https://developers.google.com/protocol-buffers. |=== ==== Optionally building documentation diff --git a/doc/user-guide/overview.adoc.in b/doc/user-guide/overview.adoc.in index 4c5ab55..8123173 100644 --- a/doc/user-guide/overview.adoc.in +++ b/doc/user-guide/overview.adoc.in @@ -21,9 +21,9 @@ The DNS-STATS Compactor suite currently comprises two programs: * _compactor_. Similar in usage to the well-known http://www.tcpdump.org[_tcpdump_] utility, _compactor_ reads traffic - from one or more network interfaces and writes selected details to + from one or more network interfaces (or a DNSTAP socket) and writes selected details to C-DNS and PCAP output files. _compactor_ can also read and convert - pre-recorded PCAP files. + pre-recorded PCAP files or DNSTAP files. * _inspector_. Reconstructs network traffic from C-DNS files produced by _compactor_. It outputs one or more PCAP files suitable for direct inspection or input to existing analysis tools. See @@ -44,9 +44,12 @@ The output file types that may be produced are: ancillary information, e.g. ICMP and TCP Reset counts. These files are significantly smaller than PCAP files containing the same traffic. See <>. -* 'Ignored' traffic. These contain captured non-DNS and malformed DNS packets in PCAP format. -* 'Raw' traffic. These contain all packets in the captured traffic in - PCAP format. They are similar to files produced by _tcpdump_. +* 'Ignored' traffic. When capturing from the network, these contain captured + non-DNS and malformed DNS packets in PCAP format. _Ignored_ traffic is not + available when capturing from DNSTAP. +* 'Raw' traffic. When capturing from the network, these contain all packets in + the captured traffic in PCAP format. They are similar to files produced by + _tcpdump_. _Raw_ traffic is not available when capturing from DNSTAP. === C-DNS Format @@ -105,12 +108,14 @@ with the following changes: ** _BlockPreamble_: *** _compactor-end-time_ (-1): if the block rolled over (i.e. a new incoming data caused collection to begin a new block), the timestamp of that new data is recorded as - the end time of the (older) block. If collection from an interface stops, the time - collection stops is recorded as the end time of the block. + the end time of the (older) block. If collection from an interface or DNSTAP stops, the time + collection stops is recorded as the end time of the block. No end + time is recorded when reading a PCAP or DNSTAP file. *** _compactor-start-time_ (-2): if the block rolled over (i.e. a new incoming data caused collection to begin a new block), the timestamp of that new data is recorded as - the start time of the (newer) block. If collection from an interface starts, the time - collection starts is recorded as the start time of the block. + the start time of the (newer) block. If collection from an interface or DNSTAP starts, the time + collection starts is recorded as the start time of the block. No start + time is recorded when reading a PCAP or DNSTAP file. ** _BlockStatistics_: *** _compactor-non-dns-packets_ (-1): count of the number of received packets that could not be interpreted as DNS packets. @@ -138,8 +143,9 @@ with the following changes: The current release does not support the following facilities defined in the RFC: * Malformed packet data recorded directly into C-DNS. -* qr-type field -* response-processing-data field +* `response-processing-data` field + +In addition, note that the `qr-type` field is only present when reading from DNSTAP. ==== ===== Changes in version 1.0 DNS-STATS Compactor diff --git a/doc/user-guide/running.adoc.in b/doc/user-guide/running.adoc.in index e64fcdb..a594226 100644 --- a/doc/user-guide/running.adoc.in +++ b/doc/user-guide/running.adoc.in @@ -8,7 +8,7 @@ configuration has been set up _compactor_ can be run as a service or daemon. ==== Linux with `systemd` By default, http://releases.ubuntu.com/16.04/[Ubuntu 16.04 LTS 'Xenial -Xerus'] uses `systemd`. +Xerus'] and later releases use `systemd`. _compactor_ service is a standard `systemd` service. It can be manually started, stopped and requested to reload its configuration. @@ -454,10 +454,16 @@ not get rotated and/or compressed until a) in the case of the CDNS output another DNS packet is seen or b) in the case of PCAP output any packet is seen. Alternatively a C-DNS file rotation can be forced by sending a SIGUSR1 to -`compactor`, but it is recommended to use either the configuration parameters -OR an external signal to trigger file rotation (to avoid a race condition). The -file rotation mechanism is under review and may be further updated in a coming -release. +`compactor`, however it is recommended to either + +. rely on just the file rotation configuration parameters OR +. to override the internally triggered rotation with an external signal sent at +a period much shorter then the file-rotation-period (which defaults to 300s) + +The second mechanism must be used with care to avoid a race condition because +the internal triggering cannot currently be disabled if the filename contains a +timestamp. The file rotation mechanism is under review and may be further +updated in a coming release. ==== diff --git a/src/baseoutputwriter.hpp b/src/baseoutputwriter.hpp index aa7e9cb..ca1a5ef 100644 --- a/src/baseoutputwriter.hpp +++ b/src/baseoutputwriter.hpp @@ -63,7 +63,8 @@ class BaseOutputWriter /** * \brief See if the output file needs rotating. * - * \param timestamp the time point to check for rotation. + * \param timestamp the time point to check for rotation. + * \param force force a file rotation */ virtual void checkForRotation(const std::chrono::system_clock::time_point& timestamp, bool force = false) = 0; diff --git a/src/blockcborwriter.hpp b/src/blockcborwriter.hpp index f8894fb..5dff99f 100644 --- a/src/blockcborwriter.hpp +++ b/src/blockcborwriter.hpp @@ -73,7 +73,8 @@ class BlockCborWriter : public BaseOutputWriter /** * \brief See if the output file needs rotating. * - * \param timestamp the time point to check for rotation. + * \param timestamp the time point to check for rotation. + * \param force force a file rotation */ virtual void checkForRotation(const std::chrono::system_clock::time_point& timestamp, bool force); diff --git a/src/compactor.cpp b/src/compactor.cpp index 2cccd20..e1ff47e 100644 --- a/src/compactor.cpp +++ b/src/compactor.cpp @@ -504,13 +504,13 @@ static void sniff_loop(BaseSniffers* sniffer, << sniffer_stats.pkts_dropped - last_sniffer_stats.pkts_dropped << "/" << std::setw(w) << sniffer_stats.channel_length; LOG_INFO << " Matcher : recv/dropped/queue " << std::setw(w) - << (stats.raw_packet_count - last_stats.raw_packet_count) << "/" << std::setw(w) + << stats.raw_packet_count - last_stats.raw_packet_count << "/" << std::setw(w) << stats.matcher_drop_count - last_stats.matcher_drop_count << "/" << std::setw(w) << matcher.get_length(); const char* sampling_text = sampling? "ON":"OFF"; if (config.sampling_rate > 0) { LOG_INFO << " Sampling: recv/discard/state " << std::setw(w) - << (stats.raw_packet_count - last_stats.raw_packet_count) << "/" << std::setw(w) + << stats.raw_packet_count - last_stats.raw_packet_count << "/" << std::setw(w) << stats.discarded_sampling_count - last_stats.discarded_sampling_count << "/" << std::setw(w) << sampling_text; } @@ -556,12 +556,14 @@ static void sniff_loop(BaseSniffers* sniffer, * \param matcher the query/response matcher to use. * \param config the current configuration. * \param stats collect packet statistics here. + * \param output the output channels. */ static void tap_loop(DnsTap& dnstap, std::iostream& stream, QueryResponseMatcher& matcher, const Configuration& config, - PacketStatistics& stats) + PacketStatistics& stats, + OutputChannels& output) { cno::system_clock::time_point last_recv_timestamp; cno::system_clock::time_point next_statslog_timestamp; @@ -589,14 +591,28 @@ static void tap_loop(DnsTap& dnstap, } else if ( next_statslog_timestamp <= last_recv_timestamp ) { + int w = 10; //output width, big enough for interval numbers up to 5 billion pps cno::seconds period = cno::duration_cast(last_recv_timestamp - last_statslog_timestamp); - LOG_INFO << "*Stats interval: average rate " << std::setw(10) - << (stats.raw_packet_count - last_stats.raw_packet_count) / period.count() << " pps over " + LOG_INFO << "*Stats interval: average rate " << std::setw(w) + << stats.raw_packet_count - last_stats.raw_packet_count / period.count() << " pps over " << config.log_network_stats_period << "s"; - LOG_INFO << " C-DNS : recv/dropped " << std::setw(10) - << stats.processed_message_count - last_stats.processed_message_count << "/" << std::setw(10) - << stats.output_cbor_drop_count - last_stats.output_cbor_drop_count << "/" << std::setw(10); + LOG_INFO << " Matcher : recv/dropped/queue " << std::setw(w) + << stats.raw_packet_count - last_stats.raw_packet_count << "/" << std::setw(w) + << stats.matcher_drop_count - last_stats.matcher_drop_count << "/" << std::setw(w) + << matcher.get_length(); + LOG_INFO << " CDNS : recv/dropped/queue " << std::setw(w) + << stats.processed_message_count - last_stats.processed_message_count << "/" << std::setw(w) + << stats.output_cbor_drop_count - last_stats.output_cbor_drop_count << "/" << std::setw(w) + << output.cbor->get_length(); + uint64_t cdns_written = (stats.processed_message_count - last_stats.processed_message_count) - + (stats.output_cbor_drop_count - last_stats.output_cbor_drop_count); + int tp = std::lround(cdns_written * 100.0 / (stats.processed_message_count - last_stats.processed_message_count)); + LOG_INFO << " CDNS out: writ/% traffic " << std::setw(w) + << cdns_written << "/" << std::setw(w) + << std::min(tp, 100) << "/" << std::setw(w) + << ""; + next_statslog_timestamp = last_recv_timestamp + cno::seconds(config.log_network_stats_period); last_statslog_timestamp = last_recv_timestamp; last_stats = stats; @@ -657,6 +673,7 @@ static int run_configuration(const po::variables_map& vm, // Output channels for this run. OutputChannels output; bool live_capture = false; + bool collect_cbor = false; // Signal handling. SignalHandler signal_handler({SIGPIPE, SIGINT, SIGTERM, SIGHUP, SIGUSR1}); @@ -698,6 +715,7 @@ static int run_configuration(const po::variables_map& vm, std::unique_ptr cbor = make_unique(config, std::move(encoder), live_capture); threads.emplace_back(cbor_writer, std::move(cbor), output.cbor); + collect_cbor = true; } SniffersConfiguration sniff_config; @@ -774,16 +792,22 @@ static int run_configuration(const po::variables_map& vm, [&](int signal) { signal_received = signal; - dnstap.breakloop(); - acceptor.cancel(); - service.stop(); - ::pthread_kill(my_thread, SIGUSR2); + if (signal_received != SIGUSR1) { + dnstap.breakloop(); + acceptor.cancel(); + service.stop(); + ::pthread_kill(my_thread, SIGUSR2); + } else if ( collect_cbor ) { + LOG_INFO << "Forcing C-DNS file rotation on SIGUSR1"; + CborItem empty_cbi; + output.cbor->put(empty_cbi, true); + } }); std::function handle_accept = [&](const boost::system::error_code&) { if ( signal_received == 0 ) - tap_loop(dnstap, stream, matcher, config, stats); + tap_loop(dnstap, stream, matcher, config, stats, output); acceptor.async_accept(*stream.rdbuf(), handle_accept); }; acceptor.async_accept(*stream.rdbuf(), handle_accept); @@ -802,10 +826,10 @@ static int run_configuration(const po::variables_map& vm, LOG_INFO << "Signal handler: Received - " << strsignal(signal_received); if (signal_received != SIGUSR1) sniffer.breakloop(); - else { - LOG_INFO << "Forcing C-DNS file rotation on SIGUSR1"; - CborItem empty_cbi; - output.cbor->put(empty_cbi, true); + else if ( collect_cbor ) { + LOG_INFO << "Forcing C-DNS file rotation on SIGUSR1"; + CborItem empty_cbi; + output.cbor->put(empty_cbi, true); } }); sniff_loop(&sniffer, matcher, output, config, stats); @@ -827,7 +851,7 @@ static int run_configuration(const po::variables_map& vm, signal_received = signal; dnstap.breakloop(); }); - tap_loop(dnstap, stream, matcher, config, stats); + tap_loop(dnstap, stream, matcher, config, stats, output); } else std::cerr << "Failed to open " << fname << std::endl;