add integration examples (#740)

* add influxdb
* add prometheus
* add loki

README.md

@@ -114,6 +114,9 @@ The [`_integration`](./docs/_integration) folder contains DNS-collector `configu
 - [Fluentd](./docs/_integration/fluentd/README.md)
 - [Elasticsearch](./docs/_integration/elasticsearch/README.md)
 - [Kafka](./docs/_integration/kafka/README.md)
+- [InfluxDB](./docs/_integration/influxdb/README.md)
+- [Prometheus](./docs/_integration/prometheus/README.md)
+- [Loki](./docs/_integration/loki/README.md)
 
 ## Telemetry
 

docs/_integration/elasticsearch/config.yml

@@ -8,7 +8,6 @@ pipelines:
     dnstap:
       listen-ip: 0.0.0.0
       listen-port: 6000
-      chan-buffer-size: 4096
     routing-policy:
       forward: [ elastic]
       dropped: []
@@ -17,7 +16,6 @@ pipelines:
     elasticsearch:
       server: "http://192.168.1.220:9200/"
       index:  "dnscollector"
-      chan-buffer-size: 4096
       bulk-size: 5242880
       flush-interval: 10
       compression: gzip

docs/_integration/elasticsearch/docker-compose.yml

@@ -1,4 +1,4 @@
-version: "3.8"
+
 services:
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:8.12.2

docs/_integration/fluentd/config.yml

@@ -8,7 +8,6 @@ pipelines:
     dnstap:
       listen-ip: 0.0.0.0
       listen-port: 6000
-      chan-buffer-size: 4096
     routing-policy:
       forward: [ fluentd ]
       dropped: []
@@ -27,5 +26,4 @@ pipelines:
       ca-file: ""
       cert-file: ""
       key-file: ""
-      buffer-size: 100
-      chan-buffer-size: 4096
+      buffer-size: 100

docs/_integration/fluentd/docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3.8"
 
 services:
   fluentd:

docs/_integration/influxdb/README.md

@@ -0,0 +1,32 @@
+
+# DNS-collector with InfluxDB
+
+- Download the [`docker-compose`](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/influxdb/docker-compose.yml) file
+
+- Create the `data` folder.
+
+    ```bash
+    mkdir -p ./data
+    ```
+
+- Start the docker stack:
+
+    ```bash
+    sudo docker compose up -d
+
+    sudo docker compose logs
+    ...
+    influxdb-1    | ts=2024-06-13T18:38:18.131480Z lvl=info msg=Listening log_id=0plj8Rp0000 service=tcp-listener transport=http addr=:8086 port=8086
+    ```
+
+- Go to http://127.0.0.1:8086 to create initial user with
+    organization: dnscollector
+    bucket: db_dns
+  Copy/paste the token in the DNScollector config.
+
+- Finally, run DNScollector from source and generate some DNS logs from your DNS server with DNStap protocol.
+
+    ```bash
+    go run . -config docs/_integration/influxdb/config.yml
+    ```
+

docs/_integration/influxdb/config.yml

@@ -0,0 +1,26 @@
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      forward: [ influxdb ]
+      dropped: []
+
+  - name: influxdb
+    influxdb:
+      server-url: "http://localhost:8086"
+      auth-token: "jgrt1gbfR4T8PQ41_SwqI58oU0dIFt5aNMqK3hfugPoMQBFl6OMGUQGBd_d6oI3Ylnh2ivvWbkqzwdCnUP-r-w=="
+      bucket: "db_dns"
+      organization: "dnscollector"
+      tls-support: false
+      tls-insecure: false
+      tls-min-version: 1.2
+      ca-file: ""
+      cert-file: ""
+      key-file: ""

docs/_integration/influxdb/docker-compose.yml

@@ -0,0 +1,14 @@
+
+services:
+  influxdb:
+    image: influxdb:2.7.6
+    user: 1000:1000
+    ports:
+      - "8086:8086"
+    environment:
+      - INFLUXDB_DB=db0
+      - INFLUXDB_ADMIN_USER=admin
+      - INFLUXDB_ADMIN_PASSWORD=badpassword
+    volumes:
+      - ./data:/var/lib/influxdb
+    restart: unless-stopped

docs/_integration/kafka/docker-compose.yml

@@ -1,4 +1,4 @@
-version: '3.8'
+
 services:
   kafka-ui:
     image: provectuslabs/kafka-ui:v0.7.1

docs/_integration/loki/README.md

@@ -0,0 +1,25 @@
+
+# DNS-collector with Loki
+
+- Download the [`docker-compose`](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/loki/docker-compose.yml) file
+
+- Create the `data` folder.
+
+    ```bash
+    mkdir -p ./data
+    ```
+
+- Start the docker stack:
+
+    ```bash
+    sudo docker compose up -d
+    ```
+
+- Finally, run DNScollector from source and generate some DNS logs from your DNS server with DNStap protocol.
+
+    ```bash
+    go run . -config docs/_integration/loki/config.yml
+    ```
+
+- Connect to the web interface of grafana through http://127.0.0.1:3000
+  Go to the menu `Explorer`, your DNS logs will be here.

docs/_integration/loki/config.yml

@@ -0,0 +1,35 @@
+
+global:
+  trace:
+    verbose: true
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      forward: [ loki ]
+      dropped: []
+
+  - name: loki
+    lokiclient:
+      server-url: "http://localhost:3100/loki/api/v1/push"
+      job-name: "dnscollector"
+      mode: "text"
+      flush-interval: 5
+      batch-size: 1048576
+      retry-interval: 10
+      text-format: ""
+      proxy-url: ""
+      tls-insecure: false
+      tls-min-version: 1.2
+      ca-file: ""
+      cert-file: ""
+      key-file: ""
+      basic-auth-login: ""
+      basic-auth-pwd: ""
+      basic-auth-pwd-file: ""
+      tenant-id: ""
+      relabel-configs: []
+      chan-buffer-size: 0

docs/_integration/loki/datasource.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    isDefault: true

docs/_integration/loki/docker-compose.yml

@@ -0,0 +1,21 @@
+
+services:
+  loki:
+    image: grafana/loki:3.0.0
+    user: "1000:1000"
+    ports:
+      - "3100:3100"
+    command: 
+     - '--config.file=/etc/loki/config.yaml'
+    volumes:
+      - ./data:/tmp/loki
+      - ./loki-config.yaml:/etc/loki/config.yaml
+
+  grafana:
+    image: grafana/grafana:11.0.0
+    user: "1000:1000"
+    ports:
+      - "3000:3000/tcp"
+    volumes:
+      - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml
+      - ./grafana.ini:/etc/grafana/grafana.ini

docs/_integration/loki/grafana.ini

@@ -0,0 +1,3 @@
+[security]
+admin_user = admin
+admin_password = badpassword

docs/_integration/loki/loki-config.yaml

@@ -0,0 +1,44 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+frontend:
+  encoding: protobuf
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+limits_config:
+  ingestion_rate_mb: 16
+  allow_structured_metadata: false
+
+analytics:
+  reporting_enabled: false

docs/_integration/prometheus/README.md

@@ -0,0 +1,24 @@
+
+# DNS-collector with Prometheus
+
+- Download the [`docker-compose`](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/prometheus/docker-compose.yml) file
+
+- Create the `data` folder.
+
+    ```bash
+    mkdir -p ./data
+    ```
+
+- Configure targets on prometheus.yml with IP of your DNScollector and start the docker stack:
+
+    ```bash
+    sudo docker compose up -d
+    ```
+
+- Finally, run DNScollector from source and generate some DNS logs from your DNS server with DNStap protocol.
+
+    ```bash
+    go run . -config docs/_integration/prometheus/config.yml
+    ```
+
+- Import build-in dashboards

docs/_integration/prometheus/config.yml

@@ -0,0 +1,61 @@
+
+global:
+  trace:
+    verbose: true
+  telemetry:
+    enabled: true
+    web-path: "/metrics"
+    web-listen: ":9165"
+    prometheus-prefix: "dnscollector_exporter"
+    tls-support: false
+    tls-cert-file: ""
+    tls-key-file: ""
+    client-ca-file: ""
+    basic-auth-enable: false
+    basic-auth-login: admin
+    basic-auth-pwd: changeme
+
+pipelines:
+  - name: tap
+    dnstap:
+      listen-ip: 0.0.0.0
+      listen-port: 6000
+    routing-policy:
+      forward: [ prom ]
+      dropped: []
+
+  - name: prom
+    prometheus:
+      listen-ip: 0.0.0.0
+      listen-port: 8081
+      basic-auth-enable: false
+      basic-auth-login: admin
+      basic-auth-pwd: changeme
+      tls-support: false
+      tls-mutual: false
+      tls-min-version: 1.2
+      cert-file: ""
+      key-file: ""
+      prometheus-prefix: "dnscollector"
+      top-n: 10
+      chan-buffer-size: 0
+      histogram-metrics-enabled: false
+      requesters-metrics-enabled: true
+      domains-metrics-enabled: true
+      noerror-metrics-enabled: true
+      servfail-metrics-enabled: true
+      nonexistent-metrics-enabled: true
+      timeout-metrics-enabled: true
+      prometheus-labels: ["stream_id"]
+      requesters-cache-size: 250000
+      requesters-cache-ttl: 3600
+      domains-cache-size: 500000
+      domains-cache-ttl: 3600
+      noerror-domains-cache-size: 100000
+      noerror-domains-cache-ttl: 3600
+      servfail-domains-cache-size: 10000
+      servfail-domains-cache-ttl: 3600
+      nonexistent-domains-cache-size: 10000
+      nonexistent-domains-cache-ttl: 3600
+      default-domains-cache-size: 1000
+      default-domains-cache-ttl: 3600

docs/_integration/prometheus/docker-compose.yml

@@ -0,0 +1,16 @@
+
+services:
+
+  prometheus:
+    image: prom/prometheus:v2.52.0
+    user: "1000:1000"
+    ports:
+      - "9090:9090/tcp"
+    volumes:
+      - ./data:/prometheus
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=365d'
+      - '--web.enable-lifecycle'