From f495c3c85dbdddd205f9fca0f5d9b3a3861ff2be Mon Sep 17 00:00:00 2001 From: Denis Machard <5562930+dmachard@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:58:27 +0200 Subject: [PATCH] add integration examples (#740) * add influxdb * add prometheus * add loki --- README.md | 3 + docs/_integration/elasticsearch/config.yml | 2 - .../elasticsearch/docker-compose.yml | 2 +- docs/_integration/fluentd/config.yml | 4 +- docs/_integration/fluentd/docker-compose.yml | 1 - docs/_integration/influxdb/README.md | 32 ++++++++++ docs/_integration/influxdb/config.yml | 26 ++++++++ docs/_integration/influxdb/docker-compose.yml | 14 +++++ docs/_integration/kafka/docker-compose.yml | 2 +- docs/_integration/loki/README.md | 25 ++++++++ docs/_integration/loki/config.yml | 35 +++++++++++ docs/_integration/loki/datasource.yml | 8 +++ docs/_integration/loki/docker-compose.yml | 21 +++++++ docs/_integration/loki/grafana.ini | 3 + docs/_integration/loki/loki-config.yaml | 44 +++++++++++++ docs/_integration/prometheus/README.md | 24 ++++++++ docs/_integration/prometheus/config.yml | 61 +++++++++++++++++++ .../prometheus/docker-compose.yml | 16 +++++ docs/_integration/prometheus/prometheus.yml | 13 ++++ docs/loggers/logger_prometheus.md | 8 +-- 20 files changed, 332 insertions(+), 12 deletions(-) create mode 100644 docs/_integration/influxdb/README.md create mode 100644 docs/_integration/influxdb/config.yml create mode 100644 docs/_integration/influxdb/docker-compose.yml create mode 100644 docs/_integration/loki/README.md create mode 100644 docs/_integration/loki/config.yml create mode 100644 docs/_integration/loki/datasource.yml create mode 100644 docs/_integration/loki/docker-compose.yml create mode 100644 docs/_integration/loki/grafana.ini create mode 100644 docs/_integration/loki/loki-config.yaml create mode 100644 docs/_integration/prometheus/README.md create mode 100644 docs/_integration/prometheus/config.yml create mode 100644 docs/_integration/prometheus/docker-compose.yml create mode 100644 docs/_integration/prometheus/prometheus.yml diff --git a/README.md b/README.md index e1d74bb0..c5c9245e 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,9 @@ The [`_integration`](./docs/_integration) folder contains DNS-collector `configu - [Fluentd](./docs/_integration/fluentd/README.md) - [Elasticsearch](./docs/_integration/elasticsearch/README.md) - [Kafka](./docs/_integration/kafka/README.md) +- [InfluxDB](./docs/_integration/influxdb/README.md) +- [Prometheus](./docs/_integration/prometheus/README.md) +- [Loki](./docs/_integration/loki/README.md) ## Telemetry diff --git a/docs/_integration/elasticsearch/config.yml b/docs/_integration/elasticsearch/config.yml index 78e86fef..36b6474f 100644 --- a/docs/_integration/elasticsearch/config.yml +++ b/docs/_integration/elasticsearch/config.yml @@ -8,7 +8,6 @@ pipelines: dnstap: listen-ip: 0.0.0.0 listen-port: 6000 - chan-buffer-size: 4096 routing-policy: forward: [ elastic] dropped: [] @@ -17,7 +16,6 @@ pipelines: elasticsearch: server: "http://192.168.1.220:9200/" index: "dnscollector" - chan-buffer-size: 4096 bulk-size: 5242880 flush-interval: 10 compression: gzip diff --git a/docs/_integration/elasticsearch/docker-compose.yml b/docs/_integration/elasticsearch/docker-compose.yml index d984bfe3..7320b135 100644 --- a/docs/_integration/elasticsearch/docker-compose.yml +++ b/docs/_integration/elasticsearch/docker-compose.yml @@ -1,4 +1,4 @@ -version: "3.8" + services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.12.2 diff --git a/docs/_integration/fluentd/config.yml b/docs/_integration/fluentd/config.yml index 9d885a6a..4bf0cf9a 100644 --- a/docs/_integration/fluentd/config.yml +++ b/docs/_integration/fluentd/config.yml @@ -8,7 +8,6 @@ pipelines: dnstap: listen-ip: 0.0.0.0 listen-port: 6000 - chan-buffer-size: 4096 routing-policy: forward: [ fluentd ] dropped: [] @@ -27,5 +26,4 @@ pipelines: ca-file: "" cert-file: "" key-file: "" - buffer-size: 100 - chan-buffer-size: 4096 \ No newline at end of file + buffer-size: 100 \ No newline at end of file diff --git a/docs/_integration/fluentd/docker-compose.yml b/docs/_integration/fluentd/docker-compose.yml index 104815e6..9b1067ca 100644 --- a/docs/_integration/fluentd/docker-compose.yml +++ b/docs/_integration/fluentd/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.8" services: fluentd: diff --git a/docs/_integration/influxdb/README.md b/docs/_integration/influxdb/README.md new file mode 100644 index 00000000..13f4d43a --- /dev/null +++ b/docs/_integration/influxdb/README.md @@ -0,0 +1,32 @@ + +# DNS-collector with InfluxDB + +- Download the [`docker-compose`](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/influxdb/docker-compose.yml) file + +- Create the `data` folder. + + ```bash + mkdir -p ./data + ``` + +- Start the docker stack: + + ```bash + sudo docker compose up -d + + sudo docker compose logs + ... + influxdb-1 | ts=2024-06-13T18:38:18.131480Z lvl=info msg=Listening log_id=0plj8Rp0000 service=tcp-listener transport=http addr=:8086 port=8086 + ``` + +- Go to http://127.0.0.1:8086 to create initial user with + organization: dnscollector + bucket: db_dns + Copy/paste the token in the DNScollector config. + +- Finally, run DNScollector from source and generate some DNS logs from your DNS server with DNStap protocol. + + ```bash + go run . -config docs/_integration/influxdb/config.yml + ``` + diff --git a/docs/_integration/influxdb/config.yml b/docs/_integration/influxdb/config.yml new file mode 100644 index 00000000..1427361c --- /dev/null +++ b/docs/_integration/influxdb/config.yml @@ -0,0 +1,26 @@ + +global: + trace: + verbose: true + +pipelines: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + routing-policy: + forward: [ influxdb ] + dropped: [] + + - name: influxdb + influxdb: + server-url: "http://localhost:8086" + auth-token: "jgrt1gbfR4T8PQ41_SwqI58oU0dIFt5aNMqK3hfugPoMQBFl6OMGUQGBd_d6oI3Ylnh2ivvWbkqzwdCnUP-r-w==" + bucket: "db_dns" + organization: "dnscollector" + tls-support: false + tls-insecure: false + tls-min-version: 1.2 + ca-file: "" + cert-file: "" + key-file: "" \ No newline at end of file diff --git a/docs/_integration/influxdb/docker-compose.yml b/docs/_integration/influxdb/docker-compose.yml new file mode 100644 index 00000000..00176edc --- /dev/null +++ b/docs/_integration/influxdb/docker-compose.yml @@ -0,0 +1,14 @@ + +services: + influxdb: + image: influxdb:2.7.6 + user: 1000:1000 + ports: + - "8086:8086" + environment: + - INFLUXDB_DB=db0 + - INFLUXDB_ADMIN_USER=admin + - INFLUXDB_ADMIN_PASSWORD=badpassword + volumes: + - ./data:/var/lib/influxdb + restart: unless-stopped \ No newline at end of file diff --git a/docs/_integration/kafka/docker-compose.yml b/docs/_integration/kafka/docker-compose.yml index cf6354ac..1a459032 100644 --- a/docs/_integration/kafka/docker-compose.yml +++ b/docs/_integration/kafka/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.8' + services: kafka-ui: image: provectuslabs/kafka-ui:v0.7.1 diff --git a/docs/_integration/loki/README.md b/docs/_integration/loki/README.md new file mode 100644 index 00000000..70a42a0e --- /dev/null +++ b/docs/_integration/loki/README.md @@ -0,0 +1,25 @@ + +# DNS-collector with Loki + +- Download the [`docker-compose`](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/loki/docker-compose.yml) file + +- Create the `data` folder. + + ```bash + mkdir -p ./data + ``` + +- Start the docker stack: + + ```bash + sudo docker compose up -d + ``` + +- Finally, run DNScollector from source and generate some DNS logs from your DNS server with DNStap protocol. + + ```bash + go run . -config docs/_integration/loki/config.yml + ``` + +- Connect to the web interface of grafana through http://127.0.0.1:3000 + Go to the menu `Explorer`, your DNS logs will be here. diff --git a/docs/_integration/loki/config.yml b/docs/_integration/loki/config.yml new file mode 100644 index 00000000..02d7c498 --- /dev/null +++ b/docs/_integration/loki/config.yml @@ -0,0 +1,35 @@ + +global: + trace: + verbose: true + +pipelines: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + routing-policy: + forward: [ loki ] + dropped: [] + + - name: loki + lokiclient: + server-url: "http://localhost:3100/loki/api/v1/push" + job-name: "dnscollector" + mode: "text" + flush-interval: 5 + batch-size: 1048576 + retry-interval: 10 + text-format: "" + proxy-url: "" + tls-insecure: false + tls-min-version: 1.2 + ca-file: "" + cert-file: "" + key-file: "" + basic-auth-login: "" + basic-auth-pwd: "" + basic-auth-pwd-file: "" + tenant-id: "" + relabel-configs: [] + chan-buffer-size: 0 \ No newline at end of file diff --git a/docs/_integration/loki/datasource.yml b/docs/_integration/loki/datasource.yml new file mode 100644 index 00000000..f3bd87c1 --- /dev/null +++ b/docs/_integration/loki/datasource.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + isDefault: true \ No newline at end of file diff --git a/docs/_integration/loki/docker-compose.yml b/docs/_integration/loki/docker-compose.yml new file mode 100644 index 00000000..a14ef352 --- /dev/null +++ b/docs/_integration/loki/docker-compose.yml @@ -0,0 +1,21 @@ + +services: + loki: + image: grafana/loki:3.0.0 + user: "1000:1000" + ports: + - "3100:3100" + command: + - '--config.file=/etc/loki/config.yaml' + volumes: + - ./data:/tmp/loki + - ./loki-config.yaml:/etc/loki/config.yaml + + grafana: + image: grafana/grafana:11.0.0 + user: "1000:1000" + ports: + - "3000:3000/tcp" + volumes: + - ./datasource.yml:/etc/grafana/provisioning/datasources/datasource.yml + - ./grafana.ini:/etc/grafana/grafana.ini \ No newline at end of file diff --git a/docs/_integration/loki/grafana.ini b/docs/_integration/loki/grafana.ini new file mode 100644 index 00000000..24682c50 --- /dev/null +++ b/docs/_integration/loki/grafana.ini @@ -0,0 +1,3 @@ +[security] +admin_user = admin +admin_password = badpassword \ No newline at end of file diff --git a/docs/_integration/loki/loki-config.yaml b/docs/_integration/loki/loki-config.yaml new file mode 100644 index 00000000..b13de7d2 --- /dev/null +++ b/docs/_integration/loki/loki-config.yaml @@ -0,0 +1,44 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +frontend: + encoding: protobuf + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + ingestion_rate_mb: 16 + allow_structured_metadata: false + +analytics: + reporting_enabled: false diff --git a/docs/_integration/prometheus/README.md b/docs/_integration/prometheus/README.md new file mode 100644 index 00000000..b3ae3b34 --- /dev/null +++ b/docs/_integration/prometheus/README.md @@ -0,0 +1,24 @@ + +# DNS-collector with Prometheus + +- Download the [`docker-compose`](https://github.com/dmachard/go-dnscollector/blob/doc_atags/docs/_integration/prometheus/docker-compose.yml) file + +- Create the `data` folder. + + ```bash + mkdir -p ./data + ``` + +- Configure targets on prometheus.yml with IP of your DNScollector and start the docker stack: + + ```bash + sudo docker compose up -d + ``` + +- Finally, run DNScollector from source and generate some DNS logs from your DNS server with DNStap protocol. + + ```bash + go run . -config docs/_integration/prometheus/config.yml + ``` + +- Import build-in dashboards diff --git a/docs/_integration/prometheus/config.yml b/docs/_integration/prometheus/config.yml new file mode 100644 index 00000000..21993c6c --- /dev/null +++ b/docs/_integration/prometheus/config.yml @@ -0,0 +1,61 @@ + +global: + trace: + verbose: true + telemetry: + enabled: true + web-path: "/metrics" + web-listen: ":9165" + prometheus-prefix: "dnscollector_exporter" + tls-support: false + tls-cert-file: "" + tls-key-file: "" + client-ca-file: "" + basic-auth-enable: false + basic-auth-login: admin + basic-auth-pwd: changeme + +pipelines: + - name: tap + dnstap: + listen-ip: 0.0.0.0 + listen-port: 6000 + routing-policy: + forward: [ prom ] + dropped: [] + + - name: prom + prometheus: + listen-ip: 0.0.0.0 + listen-port: 8081 + basic-auth-enable: false + basic-auth-login: admin + basic-auth-pwd: changeme + tls-support: false + tls-mutual: false + tls-min-version: 1.2 + cert-file: "" + key-file: "" + prometheus-prefix: "dnscollector" + top-n: 10 + chan-buffer-size: 0 + histogram-metrics-enabled: false + requesters-metrics-enabled: true + domains-metrics-enabled: true + noerror-metrics-enabled: true + servfail-metrics-enabled: true + nonexistent-metrics-enabled: true + timeout-metrics-enabled: true + prometheus-labels: ["stream_id"] + requesters-cache-size: 250000 + requesters-cache-ttl: 3600 + domains-cache-size: 500000 + domains-cache-ttl: 3600 + noerror-domains-cache-size: 100000 + noerror-domains-cache-ttl: 3600 + servfail-domains-cache-size: 10000 + servfail-domains-cache-ttl: 3600 + nonexistent-domains-cache-size: 10000 + nonexistent-domains-cache-ttl: 3600 + default-domains-cache-size: 1000 + default-domains-cache-ttl: 3600 \ No newline at end of file diff --git a/docs/_integration/prometheus/docker-compose.yml b/docs/_integration/prometheus/docker-compose.yml new file mode 100644 index 00000000..4dcd2a75 --- /dev/null +++ b/docs/_integration/prometheus/docker-compose.yml @@ -0,0 +1,16 @@ + +services: + + prometheus: + image: prom/prometheus:v2.52.0 + user: "1000:1000" + ports: + - "9090:9090/tcp" + volumes: + - ./data:/prometheus + - ./prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--storage.tsdb.retention.time=365d' + - '--web.enable-lifecycle' \ No newline at end of file diff --git a/docs/_integration/prometheus/prometheus.yml b/docs/_integration/prometheus/prometheus.yml new file mode 100644 index 00000000..24d34f27 --- /dev/null +++ b/docs/_integration/prometheus/prometheus.yml @@ -0,0 +1,13 @@ +global: + scrape_interval: 15s + +scrape_configs: + - job_name: 'dnscollector_exporter' + static_configs: + - targets: ['192.168.1.16:9165'] + scrape_interval: 5s + + - job_name: 'dnscollector_traffic' + static_configs: + - targets: ['192.168.1.16:8081'] + scrape_interval: 5s \ No newline at end of file diff --git a/docs/loggers/logger_prometheus.md b/docs/loggers/logger_prometheus.md index b3d52af3..8e455f0c 100644 --- a/docs/loggers/logger_prometheus.md +++ b/docs/loggers/logger_prometheus.md @@ -101,10 +101,10 @@ prometheus: histogram-metrics-enabled: false requesters-metrics-enabled: true domains-metrics-enabled: true - noerror-domains-metrics-enabled: true - servfail-domains-metrics-enabled: true - nonexistent-domains-metrics-enabled: true - timeout-domains-metrics-enabled: true + noerror-metrics-enabled: true + servfail-metrics-enabled: true + nonexistent-metrics-enabled: true + timeout-metrics-enabled: true prometheus-labels: ["stream_id"] requesters-cache-size: 250000 requesters-cache-ttl: 3600