add tests for pipeline mode (#599)

* add tests for pipeline mode
* add examples

docs/_examples/use-case-1.pipeline.yml

@@ -0,0 +1,26 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000
+# and logging in both text and pcap formats.
+
+global:
+  trace:
+    verbose: true 
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      default: [ text, pcap ]
+
+  - name: text
+    logfile:
+      file-path:  "/tmp/dnstap.log"
+      max-size: 100
+      max-files: 10
+      mode: text
+
+  - name: pcap
+    logfile:
+      file-path: "/tmp/dns.pcap"
+      mode: pcap

docs/_examples/use-case-10.pipeline.yml

@@ -0,0 +1,21 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# and applies tranformation to reduce qname to lowercase
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: text
+    transforms:
+      normalize:
+        qname-lowercase: true

docs/_examples/use-case-11.pipeline.yml

@@ -0,0 +1,22 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# and add geographical metadata with GeoIP database
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    transforms:
+      geoip:
+        mmdb-country-file: "./testsdata/GeoLite2-Country.mmdb"
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: text
+      text-format: "localtime identity queryip qname qtype geoip-country rcode"

docs/_examples/use-case-11.yml

@@ -27,7 +27,7 @@ multiplexer:
     - name: console
       stdout:
         mode: text
-        text-format: "localtime identity queryip qname qtype rcode country"
+        text-format: "localtime identity queryip qname qtype geoip-country rcode"
 
   # Routes DNS messages from the tap collector to standard output
   routes:

docs/_examples/use-case-12.pipeline.yml

@@ -0,0 +1,24 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# and duplicate the flow to two dnstap receiver
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: relay-in
+    dnstap-relay:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      default: [ relay-out1, relay-out2 ]
+
+  - name: relay-out1
+    dnstapclient:
+      remote-address: 127.0.0.1
+      remote-port: 6001
+
+  - name: relay-out2
+    dnstapclient:
+      remote-address: 127.0.0.1
+      remote-port: 6002

docs/_examples/use-case-13.pipeline.yml

@@ -0,0 +1,20 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# and save to a file as DNStap 
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      default: [ dnstap ]
+
+  - name: dnstap
+    logfile:
+      file-path:  /tmp/dnstap.fstrm
+      flush-interval: 10
+      mode: dnstap

docs/_examples/use-case-14.pipeline.yml

@@ -0,0 +1,31 @@
+# This configuration sets up watch and read DNStap files;
+# and logging to the console in text format.
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      default: [ out-dnstap ]
+
+  - name: out-dnstap
+    logfile:
+      file-path:  /tmp/dnstap.fstrm
+      flush-interval: 1
+      mode: dnstap
+
+  - name: file-dnstap
+    file-ingestor:
+      watch-dir: /tmp
+      watch-mode: dnstap
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: text

docs/_examples/use-case-15.pipeline.yml

@@ -0,0 +1,21 @@
+# This configuration sets up watch and read PCAP files;
+# and logging to the console in JSON format.
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: pcap
+    file-ingestor:
+      watch-dir: /tmp/
+      watch-mode: pcap
+    transforms:
+      normalize:
+        qname-lowercase: true
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: json

docs/_examples/use-case-16.pipeline.yml

@@ -0,0 +1,19 @@
+# This configuration sets up DNS traffic monitoring through PowerDNS protobuf on port 6001;
+# and transforms it to DNStap on port 6002.
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: pdns
+    powerdns:
+      listen-ip: 0.0.0.0
+      listen-port: 6001
+    routing-policy:
+      default: [ tap ]
+
+  - name: tap
+    dnstapclient:
+      remote-address: 127.0.0.1
+      remote-port: 6002

docs/_examples/use-case-17.pipeline.yml

@@ -0,0 +1,19 @@
+# This configuration sets up DNS traffic monitoring through TZSP protocol on port 1000;
+# and logging to the console in JSON format.
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tzsp
+    tzsp:
+        enable: true
+        listen-ip: "0.0.0.0"
+        listen-port: 10000
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: json

docs/_examples/use-case-18.pipeline.yml

@@ -0,0 +1,28 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# applies tranformations on it and send to the console and prometheus
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    transforms:
+      latency:
+        measure-latency: false
+        unanswered-queries: true
+        queries-timeout: 2
+    routing-policy:
+      default: [ console, prom ]
+
+  - name: console
+    stdout:
+      mode: text
+
+  - name: prom
+    prometheus:
+      listen-ip: 0.0.0.0
+      listen-port: 8080

docs/_examples/use-case-19.pipeline.yml

@@ -0,0 +1,30 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# applies tranformations on dnstap collector and the console logger
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    transforms:
+      normalize:
+        qname-lowercase: true
+      latency:
+        measure-latency: true
+        queries-timeout: 2
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: text
+      text-format: timestamp-rfc3339ns identity operation rcode qname qtype latency suspicious-score
+    transforms:
+      filtering:
+        log-queries : false
+      suspicious:
+        threshold-qname-len: 4

docs/_examples/use-case-2.pipeline.yml

@@ -0,0 +1,19 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000, 
+# and computes Prometheus metrics for analysis.
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      default: [ prom ]
+
+  - name: prom
+    prometheus:
+      listen-ip: 0.0.0.0
+      listen-port: 8080

docs/_examples/use-case-20.pipeline.yml

@@ -0,0 +1,24 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# removes duplicate traffic and log to the console
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    transforms:
+      reducer:
+        repetitive-traffic-detector: true
+        qname-plus-one: false
+        watch-interval: 5
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: text
+      text-format: "timestamp-rfc3339ns identity operation rcode queryip qname qtype reducer-occurrences reducer-cumulative-length"

docs/_examples/use-case-20.yml

@@ -22,7 +22,7 @@ multiplexer:
     - name: console
       stdout:
         mode: text
-        text-format: "timestamp-rfc3339ns identity operation rcode queryip qname qtype reducer-occurences reducer-cumulative-length"
+        text-format: "timestamp-rfc3339ns identity operation rcode queryip qname qtype reducer-occurrences reducer-cumulative-length"
 
   routes:
     - from: [ tap ]

docs/_examples/use-case-21.pipeline.yml

@@ -0,0 +1,18 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# and log the console as PCAP format
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: pcap

docs/_examples/use-case-22.pipeline.yml

@@ -0,0 +1,23 @@
+# This configuration sets up DNS traffic monitoring through DNStap on port 6000;
+# applies  machine learning transformation on it
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    transforms:
+      machine-learning:
+        add-features: true
+    routing-policy:
+      default: [ console ]
+
+  - name: console
+    stdout:
+      mode: text
+      text-format: ml-size ml-entropy ml-length ml-digits ml-lowers ml-uppers
+

docs/_examples/use-case-22.yml

@@ -15,7 +15,7 @@ multiplexer:
   loggers:
     - name: console
       stdout:
-        mode: pcap
+        mode: text
         text-format: ml-size ml-entropy ml-length ml-digits ml-lowers ml-uppers
 
   routes: