Update examples

dmachard · May 30, 2024 · 29b22a8 · 29b22a8
1 parent 62df5ef
commit 29b22a8
Show file tree

Hide file tree

Showing 17 changed files with 273 additions and 210 deletions.
diff --git a/config.yml b/config.yml
@@ -32,9 +32,6 @@ pipelines:
   - name: console
     stdout:
       mode: text
-    routing-policy:
-      forward: []
-      dropped: []
 
 ################################################
 # DEPRECATED - multiplexer configuration

diff --git a/docs/_integration/elasticsearch/config.deprecated.yml b/docs/_integration/elasticsearch/config.deprecated.yml
@@ -0,0 +1,25 @@
+
+global:
+  trace:
+    verbose: true
+
+multiplexer:
+  collectors:
+    - name: tap
+      dnstap:
+        listen-ip: 0.0.0.0
+        listen-port: 6000
+        chan-buffer-size: 4096
+  loggers:
+    - name: elastic
+      elasticsearch:
+        server: "http://192.168.1.220:9200/"
+        index:  "dnscollector"
+        chan-buffer-size: 4096
+        bulk-size: 5242880
+        flush-interval: 10
+        compression: gzip
+        bulk-channel-size: 10
+  routes:
+    - from: [ tap ]
+      to: [ elastic ]
diff --git a/docs/_integration/elasticsearch/config.yml b/docs/_integration/elasticsearch/config.yml
@@ -3,23 +3,22 @@ global:
   trace:
     verbose: true
 
-multiplexer:
-  collectors:
-    - name: tap
-      dnstap:
-        listen-ip: 0.0.0.0
-        listen-port: 6000
-        chan-buffer-size: 4096
-  loggers:
-    - name: elastic
-      elasticsearch:
-        server: "http://192.168.1.220:9200/"
-        index:  "dnscollector"
-        chan-buffer-size: 4096
-        bulk-size: 5242880
-        flush-interval: 10
-        compression: gzip
-        bulk-channel-size: 10
-  routes:
-    - from: [ tap ]
-      to: [ elastic ]
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+      chan-buffer-size: 4096
+    routing-policy:
+      forward: [ elastic]
+      dropped: []
+
+  - name: elastic
+    elasticsearch:
+      server: "http://192.168.1.220:9200/"
+      index:  "dnscollector"
+      chan-buffer-size: 4096
+      bulk-size: 5242880
+      flush-interval: 10
+      compression: gzip
+      bulk-channel-size: 10
diff --git a/docs/_integration/fluentd/config.deprecated.yml b/docs/_integration/fluentd/config.deprecated.yml
@@ -0,0 +1,33 @@
+
+global:
+  trace:
+    verbose: true
+
+multiplexer:
+  collectors:
+    - name: tap
+      dnstap:
+        listen-ip: 0.0.0.0
+        listen-port: 6000
+        chan-buffer-size: 4096
+  loggers:
+    - name: fluentd
+      fluentd:
+        transport: tcp
+        remote-address: 127.0.0.1
+        remote-port: 24224
+        connect-timeout: 5
+        retry-interval: 10
+        flush-interval: 30
+        tag: "dns.collector"
+        tls-insecure: false
+        tls-min-version: 1.2
+        ca-file: ""
+        cert-file: ""
+        key-file: ""
+        buffer-size: 100
+        chan-buffer-size: 4096
+
+  routes:
+    - from: [ tap ]
+      to: [ fluentd ]
diff --git a/docs/_integration/fluentd/config.yml b/docs/_integration/fluentd/config.yml
@@ -3,31 +3,29 @@ global:
   trace:
     verbose: true
 
-multiplexer:
-  collectors:
-    - name: tap
-      dnstap:
-        listen-ip: 0.0.0.0
-        listen-port: 6000
-        chan-buffer-size: 4096
-  loggers:
-    - name: fluentd
-      fluentd:
-        transport: tcp
-        remote-address: 127.0.0.1
-        remote-port: 24224
-        connect-timeout: 5
-        retry-interval: 10
-        flush-interval: 30
-        tag: "dns.collector"
-        tls-insecure: false
-        tls-min-version: 1.2
-        ca-file: ""
-        cert-file: ""
-        key-file: ""
-        buffer-size: 100
-        chan-buffer-size: 4096
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+      chan-buffer-size: 4096
+    routing-policy:
+      forward: [ fluentd ]
+      dropped: []
 
-  routes:
-    - from: [ tap ]
-      to: [ fluentd ]
+  - name: fluentd
+    fluentd:
+      transport: tcp
+      remote-address: 127.0.0.1
+      remote-port: 24224
+      connect-timeout: 5
+      retry-interval: 10
+      flush-interval: 30
+      tag: "dns.collector"
+      tls-insecure: false
+      tls-min-version: 1.2
+      ca-file: ""
+      cert-file: ""
+      key-file: ""
+      buffer-size: 100
+      chan-buffer-size: 4096
diff --git a/docs/_integration/kafka/config.deprecated.yml b/docs/_integration/kafka/config.deprecated.yml
@@ -0,0 +1,36 @@
+
+global:
+  trace:
+    verbose: true
+
+multiplexer:
+  collectors:
+    - name: tap
+      dnstap:
+        listen-ip: 0.0.0.0
+        listen-port: 6000
+        chan-buffer-size: 4096
+  loggers:
+    - name: kafka
+      kafkaproducer:
+        remote-address: 127.0.0.1
+        remote-port: 9092
+        connect-timeout: 5
+        retry-interval: 10
+        flush-interval: 30
+        tls-support: false
+        tls-insecure: false
+        sasl-support: false
+        sasl-mechanism: PLAIN
+        sasl-username: false
+        sasl-password: false
+        mode: flat-json
+        buffer-size: 100
+        topic: "dnscollector"
+        partition: 0
+        chan-buffer-size: 4096
+        compression: none
+
+  routes:
+    - from: [ tap ]
+      to: [ kafka ]
diff --git a/docs/_integration/kafka/config.yml b/docs/_integration/kafka/config.yml
@@ -3,34 +3,32 @@ global:
   trace:
     verbose: true
 
-multiplexer:
-  collectors:
-    - name: tap
-      dnstap:
-        listen-ip: 0.0.0.0
-        listen-port: 6000
-        chan-buffer-size: 4096
-  loggers:
-    - name: kafka
-      kafkaproducer:
-        remote-address: 127.0.0.1
-        remote-port: 9092
-        connect-timeout: 5
-        retry-interval: 10
-        flush-interval: 30
-        tls-support: false
-        tls-insecure: false
-        sasl-support: false
-        sasl-mechanism: PLAIN
-        sasl-username: false
-        sasl-password: false
-        mode: flat-json
-        buffer-size: 100
-        topic: "dnscollector"
-        partition: 0
-        chan-buffer-size: 4096
-        compression: none
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+      chan-buffer-size: 4096
+    routing-policy:
+      forward: [ kafka ]
+      dropped: []
 
-  routes:
-    - from: [ tap ]
-      to: [ kafka ]
+  - name: kafka
+    kafkaproducer:
+      remote-address: 127.0.0.1
+      remote-port: 9092
+      connect-timeout: 5
+      retry-interval: 10
+      flush-interval: 30
+      tls-support: false
+      tls-insecure: false
+      sasl-support: false
+      sasl-mechanism: PLAIN
+      sasl-username: false
+      sasl-password: false
+      mode: flat-json
+      buffer-size: 100
+      topic: "dnscollector"
+      partition: 0
+      chan-buffer-size: 4096
+      compression: none
diff --git a/docs/examples.md b/docs/examples.md
@@ -3,6 +3,9 @@
 
 You will find below some examples of configurations to manage your DNS logs.
 
+- Pipelines running mode
+  - [x] [Advanced example with DNSmessage collector](./_examples/use-case-24.yml)
+
 - Capture DNS traffic from incoming DNSTap streams
   - [x] [Read from UNIX DNSTap socket and forward it to TLS stream](./_examples/use-case-5.yml)
   - [x] [Transform DNSTap as input to JSON format as output](./_examples/use-case-3.yml)

diff --git a/tests/testsdata/config_bench.yml b/tests/testsdata/config_bench.yml
@@ -2,22 +2,19 @@ global:
   trace:
     verbose: true
 
-multiplexer:
-  collectors:
-    - name: tap
-      dnstap:
-        listen-ip: 0.0.0.0
-        listen-port: 6000
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      forward: [ web ]
+      dropped: []
 
-  loggers:
-    - name: web
-      webserver:
-        listen-ip: 0.0.0.0
-        listen-port: 8080
-        top-max-items: 100
-        basic-auth-login: admin
-        basic-auth-pwd: changeme
-
-  routes:
-    - from: [tap]
-      to: [web]
+  - name: web
+    webserver:
+      listen-ip: 0.0.0.0
+      listen-port: 8080
+      top-max-items: 100
+      basic-auth-login: admin
+      basic-auth-pwd: changeme
diff --git a/tests/testsdata/config_metrics_dnstaptcp.yml b/tests/testsdata/config_metrics_dnstaptcp.yml
@@ -2,25 +2,23 @@ global:
   trace:
     verbose: true
 
-multiplexer:
-  collectors:
-    - name: tap
-      dnstap:
-        listen-ip: 0.0.0.0
-        listen-port: 6000
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      forward: [ console, api ]
+      dropped: []
 
-  loggers:
-    - name: console
-      stdout:
-        mode: text
-    - name: api
-      restapi:
-        listen-ip: 0.0.0.0
-        listen-port: 8080
-        top-n: 100
-        basic-auth-login: admin
-        basic-auth-pwd: changeme
+  - name: console
+    stdout:
+      mode: text
 
-  routes:
-    - from: [tap]
-      to: [console, api]
+  - name: api
+    restapi:
+      listen-ip: 0.0.0.0
+      listen-port: 8080
+      top-n: 100
+      basic-auth-login: admin
+      basic-auth-pwd: changeme