diff --git a/packages/stan/_dev/build/docs/README.md b/packages/stan/_dev/build/docs/README.md new file mode 100644 index 00000000000..3a0e4afba26 --- /dev/null +++ b/packages/stan/_dev/build/docs/README.md @@ -0,0 +1,50 @@ +# STAN integration + +This integration is used to collect logs and metrics from [STAN servers](https://github.com/nats-io/stan.go). +The integration collects metrics from [STAN monitoring server APIs](https://github.com/nats-io/nats-streaming-server/blob/master/server/monitor.go). + + +## Compatibility + +The STAN package is tested with Stan 0.15.1. + +## Logs + +### log + +The `log` dataset collects the STAN logs. + +{{event "log"}} + +{{fields "log"}} + +## Metrics + +The default datasets are `stats`, `channels`, and `subscriptions`. + +### stats + +This is the `stats` dataset of the STAN package, in charge of retrieving generic +metrics from a STAN instance. + +{{event "stats"}} + +{{fields "stats"}} + +### channels + +This is the `channels` dataset of the STAN package, in charge of retrieving +metrics about channels from a STAN instance. + +{{event "channels"}} + +{{fields "channels"}} + +### subscriptions + +This is the `subscriptions` dataset of the STAN package, in charge of retrieving +metrics about subscriptions from a STAN instance. + +{{event "subscriptions"}} + +{{fields "subscriptions"}} \ No newline at end of file diff --git a/packages/stan/_dev/deploy/docker/Dockerfile b/packages/stan/_dev/deploy/docker/Dockerfile new file mode 100644 index 00000000000..030427a7c00 --- /dev/null +++ b/packages/stan/_dev/deploy/docker/Dockerfile @@ -0,0 +1,26 @@ +ARG STAN_VERSION=0.15.1 +FROM nats-streaming:$STAN_VERSION + +# build stage +FROM golang:1.13-alpine3.11 AS build-env +RUN apk --no-cache add build-base git mercurial gcc +RUN cd src && go get -d github.com/nats-io/stan.go/ +RUN cd src/github.com/nats-io/stan.go/examples/stan-bench && git checkout tags/v0.5.2 && go build . + +# create an enhanced container with nc command available since nats is based +# on scratch image making healthcheck impossible +FROM alpine:latest +RUN apk add --no-cache --upgrade bash +COPY --from=0 nats-streaming-server /nats-streaming-server +COPY --from=build-env /go/src/github.com/nats-io/stan.go/examples/stan-bench/stan-bench /stan-bench +# Expose client, management, and cluster ports +EXPOSE 4222 8222 +ADD healthcheck.sh /healthcheck.sh +RUN ["chmod", "+x", "/healthcheck.sh"] +ADD run.sh /run.sh + +# Healthcheck waits until channels have been created by the benchmark that runs inside +HEALTHCHECK --interval=1s --retries=100 CMD /healthcheck.sh + +# Run with default memory based store +ENTRYPOINT ["/run.sh"] diff --git a/packages/stan/_dev/deploy/docker/docker-compose.yml b/packages/stan/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 00000000000..f94ef1bc888 --- /dev/null +++ b/packages/stan/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,11 @@ +version: '2.3' +services: + stan: + # Commented out `image:` below until we have a process to refresh the hosted images from + # Dockerfiles in this repo. Until then, we build the image locally using `build:` below. + # image: docker.elastic.co/integrations-ci/beats-stan:${SERVICE_VERSION:-2.4.20}-1 + build: . + ports: + - 8222 + volumes: + - ${SERVICE_LOGS_DIR}:/var/log/stan diff --git a/packages/stan/_dev/deploy/docker/healthcheck.sh b/packages/stan/_dev/deploy/docker/healthcheck.sh new file mode 100644 index 00000000000..28c652d9051 --- /dev/null +++ b/packages/stan/_dev/deploy/docker/healthcheck.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +res=$(wget -q -O - http://0.0.0.0:8222/streaming/channelsz | sed -n 's/"count": \([[:digit:]]\+\),/\1/p') + +if [[ $res -gt 0 ]]; then + exit 0 +fi + +exit 1 diff --git a/packages/stan/_dev/deploy/docker/run.sh b/packages/stan/_dev/deploy/docker/run.sh new file mode 100755 index 00000000000..a37326f9fea --- /dev/null +++ b/packages/stan/_dev/deploy/docker/run.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +/nats-streaming-server -DV -l /var/log/stan/stan.log -m 8222 & +sleep 2 +while true; do /stan-bench -np 0 -ns 100 -qgroup T -n 100000000 -ms 1024 foo; done +#while true; do /stan-bench -np 10 -ns 10 -n 1000000000 -ms 1024 bar; done & diff --git a/packages/stan/_dev/deploy/variants.yml b/packages/stan/_dev/deploy/variants.yml new file mode 100644 index 00000000000..17388bff92f --- /dev/null +++ b/packages/stan/_dev/deploy/variants.yml @@ -0,0 +1,4 @@ +variants: + v1: + SERVICE_VERSION: 0.15.1 +default: v1 diff --git a/packages/stan/data_stream/channels/_dev/test/system/test-default-config.yml b/packages/stan/data_stream/channels/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..83fc84f62ea --- /dev/null +++ b/packages/stan/data_stream/channels/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: + hosts: + - http://{{Hostname}}:{{Port}} +data_stream: + vars: ~ diff --git a/packages/stan/data_stream/channels/agent/stream/stream.yml.hbs b/packages/stan/data_stream/channels/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..a2408250836 --- /dev/null +++ b/packages/stan/data_stream/channels/agent/stream/stream.yml.hbs @@ -0,0 +1,6 @@ +metricsets: ["channels"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} diff --git a/packages/stan/data_stream/channels/fields/base-fields.yml b/packages/stan/data_stream/channels/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/stan/data_stream/channels/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/stan/data_stream/channels/fields/ecs.yml b/packages/stan/data_stream/channels/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/stan/data_stream/channels/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/stan/data_stream/channels/fields/fields.yml b/packages/stan/data_stream/channels/fields/fields.yml new file mode 100644 index 00000000000..f47d3ef50cb --- /dev/null +++ b/packages/stan/data_stream/channels/fields/fields.yml @@ -0,0 +1,28 @@ +- name: stan.channels + type: group + release: ga + fields: + - name: name + type: keyword + description: | + The name of the STAN streaming channel + - name: messages + type: long + description: | + The number of STAN streaming messages + - name: bytes + type: long + description: | + The number of STAN bytes in the channel + - name: first_seq + type: long + description: | + First sequence number stored in the channel. If first_seq > min([seq in subscriptions]) data loss has possibly occurred + - name: last_seq + type: long + description: | + Last sequence number stored in the channel + - name: depth + type: long + description: | + Queue depth based upon current sequence number and highest reported subscriber sequence number diff --git a/packages/stan/data_stream/channels/fields/package-fields.yml b/packages/stan/data_stream/channels/fields/package-fields.yml new file mode 100644 index 00000000000..881b7127a50 --- /dev/null +++ b/packages/stan/data_stream/channels/fields/package-fields.yml @@ -0,0 +1,11 @@ +- name: stan + type: group + fields: + - name: server.id + type: keyword + description: | + The server ID + - name: cluster.id + type: keyword + description: | + The cluster ID diff --git a/packages/stan/data_stream/channels/manifest.yml b/packages/stan/data_stream/channels/manifest.yml new file mode 100644 index 00000000000..3000a2c87e5 --- /dev/null +++ b/packages/stan/data_stream/channels/manifest.yml @@ -0,0 +1,15 @@ +type: metrics +title: Stan channels metrics +release: experimental +streams: + - input: stan/metrics + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 60s + title: Stan channels metrics + description: Collect Stan channels metrics diff --git a/packages/stan/data_stream/channels/sample_event.json b/packages/stan/data_stream/channels/sample_event.json new file mode 100644 index 00000000000..d7a04a66d5e --- /dev/null +++ b/packages/stan/data_stream/channels/sample_event.json @@ -0,0 +1,74 @@ +{ + "@timestamp": "2021-01-15T12:23:32.592Z", + "service": { + "address": "http://elastic-package-service_stan_1:8222/streaming/channelsz?subs=1", + "type": "stan" + }, + "event": { + "duration": 8406132380, + "dataset": "stan.channels", + "module": "stan" + }, + "metricset": { + "name": "channels", + "period": 60000 + }, + "stan": { + "cluster": { + "id": "test-cluster" + }, + "server": { + "id": "kvQEpbFak88fHAnWCZxZDL" + }, + "channels": { + "depth": 3966, + "name": "bar", + "messages": 4990, + "bytes": 5214423, + "first_seq": 1, + "last_seq": 4990 + } + }, + "elastic_agent": { + "version": "7.11.0", + "id": "df58bff0-5714-11eb-b094-915beebb3c66", + "snapshot": true + }, + "ecs": { + "version": "1.7.0" + }, + "data_stream": { + "type": "metrics", + "dataset": "stan.channels", + "namespace": "default" + }, + "host": { + "architecture": "x86_64", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "88c3c3ec3afebed7631b44a69754359e", + "name": "ec072aa02d8b", + "containerized": true, + "ip": [ + "172.19.0.5" + ], + "mac": [ + "02:42:ac:13:00:05" + ], + "hostname": "ec072aa02d8b" + }, + "agent": { + "version": "7.11.0", + "hostname": "ec072aa02d8b", + "ephemeral_id": "8d73aff0-201b-4260-9e89-cd519348de03", + "id": "67b9c377-7d0c-4a69-9351-2befe6386fbd", + "name": "ec072aa02d8b", + "type": "metricbeat" + } +} \ No newline at end of file diff --git a/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log b/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log new file mode 100644 index 00000000000..a900055d4b6 --- /dev/null +++ b/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log @@ -0,0 +1,69 @@ +[7] 2021/01/13 14.20:06.981022 [INF] STREAM: Starting nats-streaming-server[test-cluster] version 0.15.1 +[7] 2021/01/13 14.20:06.981055 [INF] STREAM: ServerID: tnudw5OA2pv04Sn0JpebGy +[7] 2021/01/13 14.20:06.981069 [INF] STREAM: Go version: go1.11.10 +[7] 2021/01/13 14.20:06.981082 [INF] STREAM: Git commit: [8e6aa7c] +[7] 2021/01/13 14.20:06.981951 [INF] Starting nats-server version 2.0.0 +[7] 2021/01/13 14.20:06.981983 [DBG] Go build version go1.11.10 +[7] 2021/01/13 14.20:06.981996 [INF] Git commit [not set] +[7] 2021/01/13 14.20:06.982082 [INF] Starting http monitor on 0.0.0.0:8222 +[7] 2021/01/13 14.20:06.982116 [INF] Listening for client connections on 0.0.0.0:4222 +[7] 2021/01/13 14.20:06.982127 [INF] Server id is NCE3NFAMOIJHIR6KK53KBPN6C44LXZ6G4ULGLIWIBLYOLG5Z5LVCQSOB +[7] 2021/01/13 14.20:06.982136 [INF] Server is ready +[7] 2021/01/13 14.20:06.982149 [DBG] Get non local IPs for "0.0.0.0" +[7] 2021/01/13 14.20:06.982402 [DBG] ip=192.168.240.2 +[7] 2021/01/13 14.20:07.008167 [DBG] 127.0.0.1:33150 - cid:1 - Client connection created +[7] 2021/01/13 14.20:07.009385 [DBG] 127.0.0.1:33152 - cid:2 - Client connection created +[7] 2021/01/13 14.20:07.010819 [DBG] 127.0.0.1:33154 - cid:3 - Client connection created +[7] 2021/01/13 14.20:07.011557 [INF] STREAM: Recovering the state... +[7] 2021/01/13 14.20:07.011583 [INF] STREAM: No recovered state +[7] 2021/01/13 14.20:07.263504 [INF] STREAM: Message store is MEMORY +[7] 2021/01/13 14.20:07.263623 [INF] STREAM: ---------- Store Limits ---------- +[7] 2021/01/13 14.20:07.263653 [INF] STREAM: Channels: 100 * +[7] 2021/01/13 14.20:07.263671 [INF] STREAM: --------- Channels Limits -------- +[7] 2021/01/13 14.20:07.263690 [INF] STREAM: Subscriptions: 1000 * +[7] 2021/01/13 14.20:07.263710 [INF] STREAM: Messages : 1000000 * +[7] 2021/01/13 14.20:07.263728 [INF] STREAM: Bytes : 976.56 MB * +[7] 2021/01/13 14.20:07.263749 [INF] STREAM: Age : unlimited * +[7] 2021/01/13 14.20:07.263767 [INF] STREAM: Inactivity : unlimited * +[7] 2021/01/13 14.20:07.263788 [INF] STREAM: ---------------------------------- +[7] 2021/01/13 14.20:08.988091 [DBG] 127.0.0.1:33160 - cid:4 - Client connection created +[7] 2021/01/13 14.20:09.010952 [INF] STREAM: Channel "bar" has been created +[7] 2021/01/13 14:22:50.499525 [TRC] 127.0.0.1:60748 - cid:3 - <<- [SUB _INBOX.mUz7h6B3wGNf7P7bhyQldF 43] +[7] 2021/01/13 14:22:50.497650 [DBG] 127.0.0.1:60800 - cid:27 - Client connection closed +[7] 2021/01/13 14:22:50.499560 [DBG] 127.0.0.1:60812 - cid:34 - Client connection closed +[7] 2021/01/13 14:22:50.499565 [DBG] 127.0.0.1:60818 - cid:36 - Client connection closed +[7] 2021/01/13 14:22:50.499560 [DBG] 127.0.0.1:60816 - cid:35 - Client connection closed +[7] 2021/01/13 14:22:50.499586 [TRC] 127.0.0.1:60748 - cid:3 - <<- [PING] +[7] 2021/01/13 14:22:50.499599 [DBG] 127.0.0.1:60820 - cid:38 - Client connection closed +[7] 2021/01/13 14:22:50.499607 [TRC] 127.0.0.1:60748 - cid:3 - ->> [PONG] +[7] 2021/01/13 14:22:50.499634 [DBG] 127.0.0.1:60804 - cid:30 - Client connection closed +[7] 2021/01/13 14:22:50.499651 [DBG] 127.0.0.1:60806 - cid:32 - Client connection closed +[7] 2021/01/13 14:22:50.499656 [DBG] 127.0.0.1:60814 - cid:42 - Client connection closed +[7] 2021/01/13 14:22:50.499694 [DBG] 127.0.0.1:60808 - cid:33 - Client connection closed +[7] 2021/01/13 14:22:50.499717 [DBG] 127.0.0.1:60802 - cid:31 - Client connection closed +[7] 2021/01/13 14:22:50.499727 [TRC] 127.0.0.1:60748 - cid:3 - <<- [SUB _INBOX.mUz7h6B3wGNf7P7bhyQlf6 44] +[7] 2021/01/13 14:22:50.499769 [TRC] 127.0.0.1:60748 - cid:3 - <<- [PING] +[7] 2021/01/13 14:22:50.488515 [TRC] 127.0.0.1:60910 - cid:83 - ->> [MSG _INBOX.ZXYA2FSF5VwytqOUdLsWfp.RdKmEViH 3 31] +[7] 2021/01/13 14:22:50.499782 [TRC] 127.0.0.1:60748 - cid:3 - ->> [PONG] +[7] 2021/01/13 14:22:50.499811 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsXOL.kiHXi6ux 31] +[7] 2021/01/13 14:22:50.499850 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQlS7"] +[7] 2021/01/13 14:22:50.499850 [DBG] 127.0.0.1:60910 - cid:83 - Client connection closed +[7] 2021/01/13 14:22:50.499961 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsUbq.CeeDx8Zh 31] +[7] 2021/01/13 14:22:50.500004 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQlTy"] +[7] 2021/01/13 14:22:50.500142 [DBG] 127.0.0.1:60810 - cid:29 - Client connection closed +[7] 2021/01/13 14:22:50.508653 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsUsX.7vrrWV3v 31] +[7] 2021/01/13 14:22:50.508725 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQm1M"] +[7] 2021/01/13 14:22:50.508872 [TRC] 127.0.0.1:60748 - cid:3 - <<- [SUB _INBOX.mUz7h6B3wGNf7P7bhyQm3D 57] +[7] 2021/01/13 14:22:50.509016 [TRC] 127.0.0.1:60748 - cid:3 - <<- [PING] +[7] 2021/01/13 14:22:50.509046 [TRC] 127.0.0.1:60748 - cid:3 - ->> [PONG] +[7] 2021/01/13 14:22:50.509787 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsYyH.3U691ECJ 31] +[7] 2021/01/13 14:22:50.509825 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQm3D"] +[7] 2021/01/13 14:22:50.509825 [TRC] 127.0.0.1:60748 - cid:3 - <<- [SUB _INBOX.mUz7h6B3wGNf7P7bhyQm54 58] +[7] 2021/01/13 14:22:50.509899 [TRC] 127.0.0.1:60748 - cid:3 - <<- [PING] +[7] 2021/01/13 14:22:50.509912 [TRC] 127.0.0.1:60748 - cid:3 - ->> [PONG] +[7] 2021/01/13 14:22:50.510036 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsYpN.R6hUgpF6 31] +[7] 2021/01/13 14:22:50.510073 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQm54"] +[7] 2021/01/13 14:22:50.949500 [TRC] 127.0.0.1:60746 - cid:2 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsWF7.IqlBSvhV 220] +[7] 2021/01/13 14:22:50.949675 [TRC] 127.0.0.1:60746 - cid:2 - <<- MSG_PAYLOAD: ["\n _STAN.pub.mUz7h6B3wGNf7P7bhyQkHm\x12 _STAN.sub.mUz7h6B3wGNf7P7bhyQkHm\x1a\"_STAN.unsub.mUz7h6B3wGNf7P7bhyQkHm\"\"_STAN.close.mUz7h6B3wGNf7P7bhyQkHm2%_STAN.subclose.mUz7h6B3wGNf7P7bhyQkHm:!_STAN.discover.test-cluster.pings@\x05H\x03P\x01"] +[7] 2021/01/13 14:22:50.949848 [TRC] 127.0.0.1:60748 - cid:3 - <<- [UNSUB 21 ] +[7] 2021/01/13 14:22:50.949912 [TRC] 127.0.0.1:60748 - cid:3 - <-> [DELSUB 21] \ No newline at end of file diff --git a/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log-config.json b/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log-config.json new file mode 100644 index 00000000000..f945a611272 --- /dev/null +++ b/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log-config.json @@ -0,0 +1,11 @@ +{ + "fields": { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + } + }, + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json b/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json new file mode 100644 index 00000000000..6990fce24c2 --- /dev/null +++ b/packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log-expected.json @@ -0,0 +1,1999 @@ +{ + "expected": [ + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.981022 [INF] STREAM: Starting nats-streaming-server[test-cluster] version 0.15.1", + "event": { + "ingested": "2021-01-18T13:40:04.742504300Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.981022 [INF] STREAM: Starting nats-streaming-server[test-cluster] version 0.15.1]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.981055 [INF] STREAM: ServerID: tnudw5OA2pv04Sn0JpebGy", + "event": { + "ingested": "2021-01-18T13:40:04.742522500Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.981055 [INF] STREAM: ServerID: tnudw5OA2pv04Sn0JpebGy]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.981069 [INF] STREAM: Go version: go1.11.10", + "event": { + "ingested": "2021-01-18T13:40:04.742533Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.981069 [INF] STREAM: Go version: go1.11.10]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.981082 [INF] STREAM: Git commit: [8e6aa7c]", + "event": { + "ingested": "2021-01-18T13:40:04.742543Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.981082 [INF] STREAM: Git commit: [8e6aa7c]]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.981951 [INF] Starting nats-server version 2.0.0", + "event": { + "ingested": "2021-01-18T13:40:04.742553100Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.981951 [INF] Starting nats-server version 2.0.0]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.981983 [DBG] Go build version go1.11.10", + "event": { + "ingested": "2021-01-18T13:40:04.742563200Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.981983 [DBG] Go build version go1.11.10]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.981996 [INF] Git commit [not set]", + "event": { + "ingested": "2021-01-18T13:40:04.742573200Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.981996 [INF] Git commit [not set]]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.982082 [INF] Starting http monitor on 0.0.0.0:8222", + "event": { + "ingested": "2021-01-18T13:40:04.742583100Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.982082 [INF] Starting http monitor on 0.0.0.0:8222]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.982116 [INF] Listening for client connections on 0.0.0.0:4222", + "event": { + "ingested": "2021-01-18T13:40:04.742592Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.982116 [INF] Listening for client connections on 0.0.0.0:4222]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.982127 [INF] Server id is NCE3NFAMOIJHIR6KK53KBPN6C44LXZ6G4ULGLIWIBLYOLG5Z5LVCQSOB", + "event": { + "ingested": "2021-01-18T13:40:04.742595900Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.982127 [INF] Server id is NCE3NFAMOIJHIR6KK53KBPN6C44LXZ6G4ULGLIWIBLYOLG5Z5LVCQSOB]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.982136 [INF] Server is ready", + "event": { + "ingested": "2021-01-18T13:40:04.742599400Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.982136 [INF] Server is ready]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.982149 [DBG] Get non local IPs for \"0.0.0.0\"", + "event": { + "ingested": "2021-01-18T13:40:04.742603100Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.982149 [DBG] Get non local IPs for \\\"0.0.0.0\\\"]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:06.982402 [DBG] ip=192.168.240.2", + "event": { + "ingested": "2021-01-18T13:40:04.742609700Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:06.982402 [DBG] ip=192.168.240.2]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.008167 [DBG] 127.0.0.1:33150 - cid:1 - Client connection created", + "event": { + "ingested": "2021-01-18T13:40:04.742620500Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.008167 [DBG] 127.0.0.1:33150 - cid:1 - Client connection created]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.009385 [DBG] 127.0.0.1:33152 - cid:2 - Client connection created", + "event": { + "ingested": "2021-01-18T13:40:04.742631Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.009385 [DBG] 127.0.0.1:33152 - cid:2 - Client connection created]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.010819 [DBG] 127.0.0.1:33154 - cid:3 - Client connection created", + "event": { + "ingested": "2021-01-18T13:40:04.742641100Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.010819 [DBG] 127.0.0.1:33154 - cid:3 - Client connection created]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.011557 [INF] STREAM: Recovering the state...", + "event": { + "ingested": "2021-01-18T13:40:04.742688300Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.011557 [INF] STREAM: Recovering the state...]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.011583 [INF] STREAM: No recovered state", + "event": { + "ingested": "2021-01-18T13:40:04.742696200Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.011583 [INF] STREAM: No recovered state]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263504 [INF] STREAM: Message store is MEMORY", + "event": { + "ingested": "2021-01-18T13:40:04.742703700Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263504 [INF] STREAM: Message store is MEMORY]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263623 [INF] STREAM: ---------- Store Limits ----------", + "event": { + "ingested": "2021-01-18T13:40:04.742714500Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263623 [INF] STREAM: ---------- Store Limits ----------]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263653 [INF] STREAM: Channels: 100 *", + "event": { + "ingested": "2021-01-18T13:40:04.742723200Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263653 [INF] STREAM: Channels: 100 *]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263671 [INF] STREAM: --------- Channels Limits --------", + "event": { + "ingested": "2021-01-18T13:40:04.742727200Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263671 [INF] STREAM: --------- Channels Limits --------]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263690 [INF] STREAM: Subscriptions: 1000 *", + "event": { + "ingested": "2021-01-18T13:40:04.742730800Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263690 [INF] STREAM: Subscriptions: 1000 *]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263710 [INF] STREAM: Messages : 1000000 *", + "event": { + "ingested": "2021-01-18T13:40:04.742734400Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263710 [INF] STREAM: Messages : 1000000 *]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263728 [INF] STREAM: Bytes : 976.56 MB *", + "event": { + "ingested": "2021-01-18T13:40:04.742741100Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263728 [INF] STREAM: Bytes : 976.56 MB *]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263749 [INF] STREAM: Age : unlimited *", + "event": { + "ingested": "2021-01-18T13:40:04.742751900Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263749 [INF] STREAM: Age : unlimited *]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263767 [INF] STREAM: Inactivity : unlimited *", + "event": { + "ingested": "2021-01-18T13:40:04.742757900Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263767 [INF] STREAM: Inactivity : unlimited *]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:07.263788 [INF] STREAM: ----------------------------------", + "event": { + "ingested": "2021-01-18T13:40:04.742765800Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:07.263788 [INF] STREAM: ----------------------------------]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:08.988091 [DBG] 127.0.0.1:33160 - cid:4 - Client connection created", + "event": { + "ingested": "2021-01-18T13:40:04.742853300Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:08.988091 [DBG] 127.0.0.1:33160 - cid:4 - Client connection created]" + } + }, + { + "@timestamp": "2020-04-28T11:07:58.223Z", + "ecs": { + "version": "1.5.0" + }, + "message": "[7] 2021/01/13 14.20:09.010952 [INF] STREAM: Channel \"bar\" has been created", + "event": { + "ingested": "2021-01-18T13:40:04.742865300Z" + }, + "error": { + "message": "Provided Grok expressions do not match field value: [[7] 2021/01/13 14.20:09.010952 [INF] STREAM: Channel \\\"bar\\\" has been created]" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "subscribe", + "subject": "_INBOX.mUz7h6B3wGNf7P7bhyQldF", + "sid": "43" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742877300Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.497Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "27" + } + } + }, + "client": { + "port": 60800, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742887500Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "34" + } + } + }, + "client": { + "port": 60812, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742897800Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "36" + } + } + }, + "client": { + "port": 60818, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742908300Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "35" + } + } + }, + "client": { + "port": 60816, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742918800Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "ping" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742928900Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "38" + } + } + }, + "client": { + "port": 60820, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742939100Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "pong" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742949200Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "outbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "30" + } + } + }, + "client": { + "port": 60804, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742959700Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "32" + } + } + }, + "client": { + "port": 60806, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742966200Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "42" + } + } + }, + "client": { + "port": 60814, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742973700Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "33" + } + } + }, + "client": { + "port": 60808, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742984300Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "31" + } + } + }, + "client": { + "port": 60802, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.742995300Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "subscribe", + "subject": "_INBOX.mUz7h6B3wGNf7P7bhyQlf6", + "sid": "44" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743002900Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "ping" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743013700Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.488Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "message", + "bytes": 31, + "subject": "_INBOX.ZXYA2FSF5VwytqOUdLsWfp.RdKmEViH", + "sid": "3" + }, + "client": { + "id": "83" + } + } + }, + "client": { + "port": 60910, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743039800Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "outbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "pong" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743049600Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "outbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "publish", + "bytes": 31, + "subject": "_INBOX.ZXYA2FSF5VwytqOUdLsXOL.kiHXi6ux" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743056800Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "payload" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743066900Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "83" + } + } + }, + "client": { + "port": 60910, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743076800Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.499Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "publish", + "bytes": 31, + "subject": "_INBOX.ZXYA2FSF5VwytqOUdLsUbq.CeeDx8Zh" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743084400Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.500Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "payload" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743091600Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.500Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "debug" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "29" + } + } + }, + "client": { + "port": 60810, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743098900Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Client connection closed" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.508Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "publish", + "bytes": 31, + "subject": "_INBOX.ZXYA2FSF5VwytqOUdLsUsX.7vrrWV3v" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743107200Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.508Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "payload" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743116200Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.508Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "subscribe", + "subject": "_INBOX.mUz7h6B3wGNf7P7bhyQm3D", + "sid": "57" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743150600Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.509Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "ping" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743160300Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.509Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "pong" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743253600Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "outbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.509Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "publish", + "bytes": 31, + "subject": "_INBOX.ZXYA2FSF5VwytqOUdLsYyH.3U691ECJ" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743272300Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.509Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "payload" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743284400Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.509Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "subscribe", + "subject": "_INBOX.mUz7h6B3wGNf7P7bhyQm54", + "sid": "58" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743294100Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.509Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "ping" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743300900Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.509Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "pong" + }, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743311400Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "outbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.510Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "publish", + "bytes": 31, + "subject": "_INBOX.ZXYA2FSF5VwytqOUdLsYpN.R6hUgpF6" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743318900Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.510Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "payload" + }, + "client": { + "id": "1" + } + } + }, + "client": { + "port": 60744, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743326400Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.949Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "publish", + "bytes": 220, + "subject": "_INBOX.ZXYA2FSF5VwytqOUdLsWF7.IqlBSvhV" + }, + "client": { + "id": "2" + } + } + }, + "client": { + "port": 60746, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743336500Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.949Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": { + "type": "payload" + }, + "client": { + "id": "2" + } + } + }, + "client": { + "port": 60746, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743349700Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "network": { + "direction": "inbound" + } + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.949Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743358500Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "\u003c\u003c- [UNSUB 21 ]" + }, + { + "process": { + "pid": 7 + }, + "@timestamp": "2021-01-13T14:22:50.949Z", + "ecs": { + "version": "1.5.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "log": { + "level": "trace" + }, + "stan": { + "log": { + "msg": {}, + "client": { + "id": "3" + } + } + }, + "client": { + "port": 60748, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-18T13:40:04.743365400Z", + "type": [ + "info" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "\u003c-\u003e [DELSUB 21]" + } + ] +} \ No newline at end of file diff --git a/packages/stan/data_stream/log/_dev/test/system/test-default-config.yml b/packages/stan/data_stream/log/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..9c2487cb7a6 --- /dev/null +++ b/packages/stan/data_stream/log/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: ~ +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/stan.log*" diff --git a/packages/stan/data_stream/log/agent/stream/log.yml.hbs b/packages/stan/data_stream/log/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..e6e2351f067 --- /dev/null +++ b/packages/stan/data_stream/log/agent/stream/log.yml.hbs @@ -0,0 +1,10 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 \ No newline at end of file diff --git a/packages/stan/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/stan/data_stream/log/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..7a735cdfe54 --- /dev/null +++ b/packages/stan/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,197 @@ +--- +description: Pipeline for parsing stan log logs +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- grok: + field: message + patterns: + - \[%{POSINT:process.pid}\]( %{STANTIME:stan.log.timestamp})? \[%{STANLOGLEVEL:log.level}\] + %{GREEDYDATA:stan.log.info} + pattern_definitions: + STANTIME: '%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}' + STANLOGLEVEL: (INF|DBG|WRN|ERR|FTL|TRC) + ignore_missing: true +- grok: + field: stan.log.info + patterns: + - '%{IPV4:client.ip}:%{POSINT:client.port} - cid:%{POSINT:stan.log.client.id} + - %{GREEDYDATA:stan.log.msg.info}' + - '%{GREEDYDATA:stan.log.msg.data}' + ignore_missing: true +- grok: + field: stan.log.msg.info + patterns: + - '%{STANDIRECTION:network.direction} %{STANPAYLOAD:stan.log.msg.type}: \[%{GREEDYDATA:stan.log.msg.payload}\]' + - '%{STANDIRECTION:network.direction} \[%{STANNOINFO:stan.log.msg.type}\]' + - '%{STANDIRECTION:network.direction} \[%{STANUNSUB:stan.log.msg.type}\s+%{POSINT:stan.log.msg.sid}(\s+%{POSINT:stan.log.msg.max_messages})?\]' + - '%{STANDIRECTION:network.direction} \[%{STANPUB:stan.log.msg.type}\s+%{NOTSPACE:stan.log.msg.subject}(\s+%{NOTSPACE:stan.log.msg.reply_to})?\s+%{POSINT:stan.log.msg.bytes}\]' + - '%{STANDIRECTION:network.direction} \[%{STANSUB:stan.log.msg.type}\s+%{NOTSPACE:stan.log.msg.subject}(\s+%{NOTSPACE:stan.log.msg.queue_group})?\s+%{POSINT:stan.log.msg.sid}\]' + - '%{STANDIRECTION:network.direction} \[%{STANMSG:stan.log.msg.type}\s+%{NOTSPACE:stan.log.msg.subject}\s+%{POSINT:stan.log.msg.sid}(\s+%{NOTSPACE:stan.log.msg.reply_to})?\s+%{POSINT:stan.log.msg.bytes}\]' + - '%{STANDIRECTION:network.direction} \[%{STANCONNECTION:stan.log.msg.type}\s+%{GREEDYDATA:stan.log.msg.data}\]' + - '%{STANDIRECTION:network.direction} \[%{STANERROR:stan.log.msg.type}\s+%{GREEDYDATA:stan.log.msg.error\]' + - '%{GREEDYDATA:stan.log.msg.data}' + pattern_definitions: + STANDIRECTION: (<<-|->>) + STANMSG: MSG + STANPUB: PUB + STANSUB: SUB + STANUNSUB: UNSUB + STANPAYLOAD: MSG_PAYLOAD + STANERROR: -ERROR + STANPING: PING + STANPONG: PONG + STANOK: OK + STANCONNECT: CONNECT + STANINFO: INFO + STANCONNECTION: (?:%{STANCONNECT}|%{STANINFO}) + STANNOINFO: (?:%{STANPING}|%{STANPONG}|%{STANOK}) + ignore_missing: true +- remove: + field: stan.log.info +- remove: + field: stan.log.msg.info + ignore_missing: true +- remove: + field: stan.log.msg.payload + ignore_missing: true +- remove: + field: message +- rename: + field: stan.log.msg.data + target_field: message + ignore_missing: true +- script: + lang: painless + source: |- + if (ctx.log.level == params.inf) { + ctx.log.level = params.info; + } else if (ctx.log.level == params.dbg) { + ctx.log.level = params.debug; + } else if (ctx.log.level == params.wrn) { + ctx.log.level = params.warning; + } else if (ctx.log.level == params.err) { + ctx.log.level = params.error; + } else if (ctx.log.level == params.ftl) { + ctx.log.level = params.fatal; + } else if (ctx.log.level == params.trc) { + ctx.log.level = params.trace; + } + params: + inf: INF + info: info + dbg: DBG + debug: debug + wrn: WRN + warning: warning + err: ERR + error: error + ftl: FTL + fatal: fatal + trc: TRC + trace: trace +- script: + lang: painless + source: |- + if (ctx.stan.log.msg.type == params.msg) { + ctx.stan.log.msg.type = params.message; + } else if (ctx.stan.log.msg.type == params.pub) { + ctx.stan.log.msg.type = params.publish; + } else if (ctx.stan.log.msg.type == params.sub) { + ctx.stan.log.msg.type = params.subscribe; + } else if (ctx.stan.log.msg.type == params.unsub) { + ctx.stan.log.msg.type = params.unsubscribe; + } else if (ctx.stan.log.msg.type == params.msg_payload) { + ctx.stan.log.msg.type = params.payload; + } else if (ctx.stan.log.msg.type == params.err) { + ctx.stan.log.msg.type = params.error; + } else if (ctx.stan.log.msg.type == params.pi) { + ctx.stan.log.msg.type = params.ping; + } else if (ctx.stan.log.msg.type == params.po) { + ctx.stan.log.msg.type = params.pong; + } else if (ctx.stan.log.msg.type == params.ok) { + ctx.stan.log.msg.type = params.acknowledge; + } else if (ctx.stan.log.msg.type == params.connect) { + ctx.stan.log.msg.type = params.connection; + } else if (ctx.stan.log.msg.type == params.info) { + ctx.stan.log.msg.type = params.information; + } + params: + msg: MSG + message: message + pub: PUB + publish: publish + sub: SUB + subscribe: subscribe + unsub: UNSUB + unsubscribe: unsubscribe + msg_payload: MSG_PAYLOAD + payload: payload + err: -ERROR + error: error + pi: PING + ping: ping + po: PONG + pong: pong + ok: OK + acknowledge: acknowledge + connect: CONNECT + connection: connection + info: INFO + information: information + if: ctx.stan.log.msg?.type != null +- script: + lang: painless + source: |- + if (ctx.network.direction == params.in) { + ctx.network.direction = params.inbound; + } else if (ctx.network.direction == params.out) { + ctx.network.direction = params.outbound; + } + params: + in: <<- + inbound: inbound + out: ->> + outbound: outbound + if: ctx.network?.direction != null +- rename: + field: '@timestamp' + target_field: event.created +- date: + field: stan.log.timestamp + target_field: '@timestamp' + formats: + - yyyy/MM/dd HH:mm:ss.SSSSSS +- remove: + field: stan.log.timestamp +- set: + field: event.kind + value: event +- append: + field: event.type + value: info +- append: + field: event.type + value: error + if: "ctx?.log?.level != null && (ctx.log.level == 'error' || ctx.log.level == 'fatal')" +- append: + field: related.ip + value: "{{client.ip}}" + if: "ctx?.client?.ip != null" +- convert: + ignore_missing: true + field: process.pid + type: long +- convert: + ignore_missing: true + field: client.port + type: long +- convert: + ignore_missing: true + field: stan.log.msg.bytes + type: long +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/stan/data_stream/log/fields/base-fields.yml b/packages/stan/data_stream/log/fields/base-fields.yml new file mode 100644 index 00000000000..4bd35e76013 --- /dev/null +++ b/packages/stan/data_stream/log/fields/base-fields.yml @@ -0,0 +1,23 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: input.type + description: Type of Filebeat input. + type: keyword +- name: log.file.path + description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + type: keyword +- name: log.offset + type: long + description: Offset of the entry in the log file. diff --git a/packages/stan/data_stream/log/fields/ecs.yml b/packages/stan/data_stream/log/fields/ecs.yml new file mode 100644 index 00000000000..a5e79097564 --- /dev/null +++ b/packages/stan/data_stream/log/fields/ecs.yml @@ -0,0 +1,71 @@ +- name: network + type: group + fields: + - name: direction + level: core + type: keyword + ignore_above: 1024 + description: "Direction of the network traffic.\nRecommended values are:\n * inbound\n * outbound\n * internal\n * external\n * unknown\n\nWhen mapping events from a host-based monitoring context, populate this field from the host's point of view.\nWhen mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter." + example: inbound +- name: related + type: group + fields: + - name: ip + level: extended + type: ip + description: All of the IPs seen on your event. +- name: error + type: group + fields: + - name: message + level: core + type: text + description: Error message. +- name: client + type: group + fields: + - name: ip + level: extended + type: ip + description: IP address of the client. + - name: port + level: core + type: long + description: Port of the client. +- name: event + type: group + fields: + - name: kind + description: The kind of the event. The highest categorization field in the hierarchy. + example: alert + ignore_above: 1024 + type: keyword + - name: type + description: Event type. The third categorization field in the hierarchy. + ignore_above: 1024 + type: keyword + - name: created + description: Time when the event was first read by an agent or by your pipeline. + example: '2016-05-23T08:05:34.857Z' + type: date +- name: log + type: group + fields: + - name: level + description: Log level of the log event. + type: keyword +- name: process.pid + type: long + description: Process id. +- name: ecs.version + type: keyword + description: ECS version +- name: message + level: core + type: text + description: 'For log events the message field contains the log message, optimized for viewing in a log viewer. + + For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. + + If multiple messages exist, they can be combined into one message.' + example: Hello World diff --git a/packages/stan/data_stream/log/fields/fields.yml b/packages/stan/data_stream/log/fields/fields.yml new file mode 100644 index 00000000000..803aa5fbc7c --- /dev/null +++ b/packages/stan/data_stream/log/fields/fields.yml @@ -0,0 +1,47 @@ +- name: stan.log + type: group + release: beta + fields: + - name: client + type: group + fields: + - name: id + type: integer + description: | + The id of the client + - name: msg + type: group + fields: + - name: bytes + type: long + format: bytes + description: | + Size of the payload in bytes + - name: type + type: keyword + description: | + The protocol message type + - name: subject + type: keyword + description: | + Subject name this message was received on + - name: sid + type: integer + description: | + The unique alphanumeric subscription ID of the subject + - name: reply_to + type: keyword + description: | + The inbox subject on which the publisher is listening for responses + - name: max_messages + type: integer + description: | + An optional number of messages to wait for before automatically unsubscribing + - name: error.message + type: text + description: | + Details about the error occurred + - name: queue_group + type: text + description: | + The queue group which subscriber will join diff --git a/packages/stan/data_stream/log/manifest.yml b/packages/stan/data_stream/log/manifest.yml new file mode 100644 index 00000000000..7f90c7de9ac --- /dev/null +++ b/packages/stan/data_stream/log/manifest.yml @@ -0,0 +1,17 @@ +type: logs +title: STAN logs +release: experimental +streams: + - input: logfile + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/stan/stan.log* + template_path: log.yml.hbs + title: STAN logs (log) + description: Collect STAN logs using log input diff --git a/packages/stan/data_stream/log/sample_event.json b/packages/stan/data_stream/log/sample_event.json new file mode 100644 index 00000000000..63dbd6d8177 --- /dev/null +++ b/packages/stan/data_stream/log/sample_event.json @@ -0,0 +1,92 @@ +{ + "agent": { + "hostname": "4d0d8c0f4097", + "name": "4d0d8c0f4097", + "id": "10a38439-cfb3-4e2f-b4a3-b06707eed149", + "type": "filebeat", + "ephemeral_id": "39abf1ff-8ee3-41b5-a553-0af2f121da94", + "version": "7.11.0" + }, + "process": { + "pid": 7 + }, + "log": { + "file": { + "path": "/tmp/service_logs/stan.log" + }, + "offset": 2064548, + "level": "trace" + }, + "elastic_agent": { + "id": "d5178560-572f-11eb-a5fc-9bbf29f84abb", + "version": "7.11.0", + "snapshot": true + }, + "network": { + "direction": "inbound" + }, + "input": { + "type": "log" + }, + "@timestamp": "2021-01-15T13:12:07.170Z", + "ecs": { + "version": "1.6.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "stan.log" + }, + "host": { + "hostname": "4d0d8c0f4097", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.21.0.6" + ], + "name": "4d0d8c0f4097", + "id": "88c3c3ec3afebed7631b44a69754359e", + "mac": [ + "02:42:ac:15:00:06" + ], + "architecture": "x86_64" + }, + "stan": { + "log": { + "msg": { + "reply_to": "_INBOX.1wOArhLwRni1eXXhnUaD8i.LMRmCG50", + "bytes": 79, + "subject": "_STAN.discover.test-cluster", + "type": "publish" + }, + "client": { + "id": "930" + } + } + }, + "client": { + "port": 50558, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-15T13:12:08.891587159Z", + "created": "2021-01-15T13:12:07.260Z", + "kind": "event", + "type": [ + "info" + ], + "dataset": "stan.log" + } +} \ No newline at end of file diff --git a/packages/stan/data_stream/stats/_dev/test/system/test-default-config.yml b/packages/stan/data_stream/stats/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..83fc84f62ea --- /dev/null +++ b/packages/stan/data_stream/stats/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: + hosts: + - http://{{Hostname}}:{{Port}} +data_stream: + vars: ~ diff --git a/packages/stan/data_stream/stats/agent/stream/stream.yml.hbs b/packages/stan/data_stream/stats/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..3b06dd6d47b --- /dev/null +++ b/packages/stan/data_stream/stats/agent/stream/stream.yml.hbs @@ -0,0 +1,6 @@ +metricsets: ["stats"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} diff --git a/packages/stan/data_stream/stats/fields/base-fields.yml b/packages/stan/data_stream/stats/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/stan/data_stream/stats/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/stan/data_stream/stats/fields/ecs.yml b/packages/stan/data_stream/stats/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/stan/data_stream/stats/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/stan/data_stream/stats/fields/fields.yml b/packages/stan/data_stream/stats/fields/fields.yml new file mode 100644 index 00000000000..786b4ace439 --- /dev/null +++ b/packages/stan/data_stream/stats/fields/fields.yml @@ -0,0 +1,32 @@ +- name: stan.stats + type: group + release: ga + fields: + - name: state + type: keyword + description: | + The cluster / streaming configuration state (STANDALONE, CLUSTERED) + - name: role + type: keyword + description: | + If clustered, role of this node in the cluster (Leader, Follower, Candidate) + - name: clients + type: integer + description: | + The number of STAN clients + - name: subscriptions + type: integer + description: | + The number of STAN streaming subscriptions + - name: channels + type: integer + description: | + The number of STAN channels + - name: messages + type: long + description: | + Number of messages across all STAN queues + - name: bytes + type: long + description: | + Number of bytes consumed across all STAN queues diff --git a/packages/stan/data_stream/stats/fields/package-fields.yml b/packages/stan/data_stream/stats/fields/package-fields.yml new file mode 100644 index 00000000000..881b7127a50 --- /dev/null +++ b/packages/stan/data_stream/stats/fields/package-fields.yml @@ -0,0 +1,11 @@ +- name: stan + type: group + fields: + - name: server.id + type: keyword + description: | + The server ID + - name: cluster.id + type: keyword + description: | + The cluster ID diff --git a/packages/stan/data_stream/stats/manifest.yml b/packages/stan/data_stream/stats/manifest.yml new file mode 100644 index 00000000000..37f677c9bf8 --- /dev/null +++ b/packages/stan/data_stream/stats/manifest.yml @@ -0,0 +1,15 @@ +type: metrics +title: Stan stats metrics +release: experimental +streams: + - input: stan/metrics + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 60s + title: Stan stats metrics + description: Collect Stan stats metrics diff --git a/packages/stan/data_stream/stats/sample_event.json b/packages/stan/data_stream/stats/sample_event.json new file mode 100644 index 00000000000..c5cae7f2290 --- /dev/null +++ b/packages/stan/data_stream/stats/sample_event.json @@ -0,0 +1,74 @@ +{ + "@timestamp": "2021-01-15T12:26:32.467Z", + "service": { + "address": "http://elastic-package-service_stan_1:8222/streaming/serverz", + "type": "stan" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "stan.stats" + }, + "ecs": { + "version": "1.7.0" + }, + "host": { + "mac": [ + "02:42:ac:13:00:05" + ], + "name": "ec072aa02d8b", + "hostname": "ec072aa02d8b", + "architecture": "x86_64", + "os": { + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "platform": "centos" + }, + "id": "88c3c3ec3afebed7631b44a69754359e", + "containerized": true, + "ip": [ + "172.19.0.5" + ] + }, + "agent": { + "version": "7.11.0", + "hostname": "ec072aa02d8b", + "ephemeral_id": "8d73aff0-201b-4260-9e89-cd519348de03", + "id": "67b9c377-7d0c-4a69-9351-2befe6386fbd", + "name": "ec072aa02d8b", + "type": "metricbeat" + }, + "event": { + "dataset": "stan.stats", + "module": "stan", + "duration": 1252350 + }, + "metricset": { + "name": "stats", + "period": 60000 + }, + "stan": { + "cluster": { + "id": "test-cluster" + }, + "stats": { + "subscriptions": 4, + "channels": 4, + "messages": 4990, + "bytes": 5214423, + "state": "STANDALONE", + "clients": 100 + }, + "server": { + "id": "kvQEpbFak88fHAnWCZxZDL" + } + }, + "elastic_agent": { + "id": "df58bff0-5714-11eb-b094-915beebb3c66", + "snapshot": true, + "version": "7.11.0" + } +} \ No newline at end of file diff --git a/packages/stan/data_stream/subscriptions/_dev/test/system/test-default-config.yml b/packages/stan/data_stream/subscriptions/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..83fc84f62ea --- /dev/null +++ b/packages/stan/data_stream/subscriptions/_dev/test/system/test-default-config.yml @@ -0,0 +1,5 @@ +vars: + hosts: + - http://{{Hostname}}:{{Port}} +data_stream: + vars: ~ diff --git a/packages/stan/data_stream/subscriptions/agent/stream/stream.yml.hbs b/packages/stan/data_stream/subscriptions/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..521aa4357fc --- /dev/null +++ b/packages/stan/data_stream/subscriptions/agent/stream/stream.yml.hbs @@ -0,0 +1,6 @@ +metricsets: ["subscriptions"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} diff --git a/packages/stan/data_stream/subscriptions/fields/base-fields.yml b/packages/stan/data_stream/subscriptions/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/stan/data_stream/subscriptions/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/stan/data_stream/subscriptions/fields/ecs.yml b/packages/stan/data_stream/subscriptions/fields/ecs.yml new file mode 100644 index 00000000000..d7ce11b7c16 --- /dev/null +++ b/packages/stan/data_stream/subscriptions/fields/ecs.yml @@ -0,0 +1,9 @@ +- name: ecs.version + type: keyword + description: ECS version +- name: service.address + type: keyword + description: Service address +- name: service.type + type: keyword + description: Service type diff --git a/packages/stan/data_stream/subscriptions/fields/fields.yml b/packages/stan/data_stream/subscriptions/fields/fields.yml new file mode 100644 index 00000000000..54ab2816726 --- /dev/null +++ b/packages/stan/data_stream/subscriptions/fields/fields.yml @@ -0,0 +1,32 @@ +- name: stan.subscriptions + type: group + release: ga + fields: + - name: id + type: keyword + description: | + The name of the STAN channel subscription (client_id) + - name: channel + type: keyword + description: | + The name of the STAN channel the subscription is associated with + - name: queue + type: keyword + description: | + The name of the NATS queue that the STAN channel subscription is associated with, if any + - name: last_sent + type: long + description: | + Last known sequence number of the subscription that was acked + - name: pending + type: long + description: | + Number of pending messages from / to the subscriber + - name: offline + type: boolean + description: | + Is the subscriber marked as offline? + - name: stalled + type: boolean + description: | + Is the subscriber known to be stalled? diff --git a/packages/stan/data_stream/subscriptions/fields/package-fields.yml b/packages/stan/data_stream/subscriptions/fields/package-fields.yml new file mode 100644 index 00000000000..881b7127a50 --- /dev/null +++ b/packages/stan/data_stream/subscriptions/fields/package-fields.yml @@ -0,0 +1,11 @@ +- name: stan + type: group + fields: + - name: server.id + type: keyword + description: | + The server ID + - name: cluster.id + type: keyword + description: | + The cluster ID diff --git a/packages/stan/data_stream/subscriptions/manifest.yml b/packages/stan/data_stream/subscriptions/manifest.yml new file mode 100644 index 00000000000..c9eab51c606 --- /dev/null +++ b/packages/stan/data_stream/subscriptions/manifest.yml @@ -0,0 +1,15 @@ +type: metrics +title: Stan subscriptions metrics +release: experimental +streams: + - input: stan/metrics + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 60s + title: Stan subscriptions metrics + description: Collect Stan subscriptions metrics diff --git a/packages/stan/data_stream/subscriptions/sample_event.json b/packages/stan/data_stream/subscriptions/sample_event.json new file mode 100644 index 00000000000..eadb58f768d --- /dev/null +++ b/packages/stan/data_stream/subscriptions/sample_event.json @@ -0,0 +1,74 @@ +{ + "@timestamp": "2021-01-15T12:25:32.509Z", + "ecs": { + "version": "1.7.0" + }, + "agent": { + "ephemeral_id": "8d73aff0-201b-4260-9e89-cd519348de03", + "id": "67b9c377-7d0c-4a69-9351-2befe6386fbd", + "name": "ec072aa02d8b", + "type": "metricbeat", + "version": "7.11.0", + "hostname": "ec072aa02d8b" + }, + "metricset": { + "name": "subscriptions", + "period": 60000 + }, + "data_stream": { + "type": "metrics", + "dataset": "stan.subscriptions", + "namespace": "default" + }, + "elastic_agent": { + "version": "7.11.0", + "id": "df58bff0-5714-11eb-b094-915beebb3c66", + "snapshot": true + }, + "host": { + "architecture": "x86_64", + "name": "ec072aa02d8b", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "88c3c3ec3afebed7631b44a69754359e", + "containerized": true, + "ip": [ + "172.19.0.5" + ], + "mac": [ + "02:42:ac:13:00:05" + ], + "hostname": "ec072aa02d8b" + }, + "event": { + "dataset": "stan.subscriptions", + "module": "stan", + "duration": 935334325 + }, + "service": { + "address": "http://elastic-package-service_stan_1:8222/streaming/channelsz?subs=1", + "type": "stan" + }, + "stan": { + "subscriptions": { + "stalled": true, + "pending": 1024, + "id": "benchmark-sub-1", + "channel": "bar", + "last_sent": 1024, + "offline": true + }, + "server": { + "id": "kvQEpbFak88fHAnWCZxZDL" + }, + "cluster": { + "id": "test-cluster" + } + } +} \ No newline at end of file diff --git a/packages/stan/docs/README.md b/packages/stan/docs/README.md new file mode 100644 index 00000000000..2ca40d3454a --- /dev/null +++ b/packages/stan/docs/README.md @@ -0,0 +1,466 @@ +# STAN integration + +This integration is used to collect logs and metrics from [STAN servers](https://github.com/nats-io/stan.go). +The integration collects metrics from [STAN monitoring server APIs](https://github.com/nats-io/nats-streaming-server/blob/master/server/monitor.go). + + +## Compatibility + +The STAN package is tested with Stan 0.15.1. + +## Logs + +### log + +The `log` dataset collects the STAN logs. + +An example event for `log` looks as following: + +```$json +{ + "agent": { + "hostname": "4d0d8c0f4097", + "name": "4d0d8c0f4097", + "id": "10a38439-cfb3-4e2f-b4a3-b06707eed149", + "type": "filebeat", + "ephemeral_id": "39abf1ff-8ee3-41b5-a553-0af2f121da94", + "version": "7.11.0" + }, + "process": { + "pid": 7 + }, + "log": { + "file": { + "path": "/tmp/service_logs/stan.log" + }, + "offset": 2064548, + "level": "trace" + }, + "elastic_agent": { + "id": "d5178560-572f-11eb-a5fc-9bbf29f84abb", + "version": "7.11.0", + "snapshot": true + }, + "network": { + "direction": "inbound" + }, + "input": { + "type": "log" + }, + "@timestamp": "2021-01-15T13:12:07.170Z", + "ecs": { + "version": "1.6.0" + }, + "related": { + "ip": [ + "127.0.0.1" + ] + }, + "data_stream": { + "namespace": "default", + "type": "logs", + "dataset": "stan.log" + }, + "host": { + "hostname": "4d0d8c0f4097", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "name": "CentOS Linux", + "family": "redhat", + "version": "7 (Core)", + "platform": "centos" + }, + "containerized": true, + "ip": [ + "172.21.0.6" + ], + "name": "4d0d8c0f4097", + "id": "88c3c3ec3afebed7631b44a69754359e", + "mac": [ + "02:42:ac:15:00:06" + ], + "architecture": "x86_64" + }, + "stan": { + "log": { + "msg": { + "reply_to": "_INBOX.1wOArhLwRni1eXXhnUaD8i.LMRmCG50", + "bytes": 79, + "subject": "_STAN.discover.test-cluster", + "type": "publish" + }, + "client": { + "id": "930" + } + } + }, + "client": { + "port": 50558, + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2021-01-15T13:12:08.891587159Z", + "created": "2021-01-15T13:12:07.260Z", + "kind": "event", + "type": [ + "info" + ], + "dataset": "stan.log" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| client.ip | IP address of the client. | ip | +| client.port | Port of the client. | long | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | +| error.message | Error message. | text | +| event.created | Time when the event was first read by an agent or by your pipeline. | date | +| event.kind | The kind of the event. The highest categorization field in the hierarchy. | keyword | +| event.type | Event type. The third categorization field in the hierarchy. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Full path to the log file this event came from. | keyword | +| log.level | Log level of the log event. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text | +| network.direction | Direction of the network traffic. Recommended values are: * inbound * outbound * internal * external * unknown When mapping events from a host-based monitoring context, populate this field from the host's point of view. When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. | keyword | +| process.pid | Process id. | long | +| related.ip | All of the IPs seen on your event. | ip | +| stan.log.client.id | The id of the client | integer | +| stan.log.msg.bytes | Size of the payload in bytes | long | +| stan.log.msg.error.message | Details about the error occurred | text | +| stan.log.msg.max_messages | An optional number of messages to wait for before automatically unsubscribing | integer | +| stan.log.msg.queue_group | The queue group which subscriber will join | text | +| stan.log.msg.reply_to | The inbox subject on which the publisher is listening for responses | keyword | +| stan.log.msg.sid | The unique alphanumeric subscription ID of the subject | integer | +| stan.log.msg.subject | Subject name this message was received on | keyword | +| stan.log.msg.type | The protocol message type | keyword | + + +## Metrics + +The default datasets are `stats`, `channels`, and `subscriptions`. + +### stats + +This is the `stats` dataset of the STAN package, in charge of retrieving generic +metrics from a STAN instance. + +An example event for `stats` looks as following: + +```$json +{ + "@timestamp": "2021-01-15T12:26:32.467Z", + "service": { + "address": "http://elastic-package-service_stan_1:8222/streaming/serverz", + "type": "stan" + }, + "data_stream": { + "namespace": "default", + "type": "metrics", + "dataset": "stan.stats" + }, + "ecs": { + "version": "1.7.0" + }, + "host": { + "mac": [ + "02:42:ac:13:00:05" + ], + "name": "ec072aa02d8b", + "hostname": "ec072aa02d8b", + "architecture": "x86_64", + "os": { + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux", + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "platform": "centos" + }, + "id": "88c3c3ec3afebed7631b44a69754359e", + "containerized": true, + "ip": [ + "172.19.0.5" + ] + }, + "agent": { + "version": "7.11.0", + "hostname": "ec072aa02d8b", + "ephemeral_id": "8d73aff0-201b-4260-9e89-cd519348de03", + "id": "67b9c377-7d0c-4a69-9351-2befe6386fbd", + "name": "ec072aa02d8b", + "type": "metricbeat" + }, + "event": { + "dataset": "stan.stats", + "module": "stan", + "duration": 1252350 + }, + "metricset": { + "name": "stats", + "period": 60000 + }, + "stan": { + "cluster": { + "id": "test-cluster" + }, + "stats": { + "subscriptions": 4, + "channels": 4, + "messages": 4990, + "bytes": 5214423, + "state": "STANDALONE", + "clients": 100 + }, + "server": { + "id": "kvQEpbFak88fHAnWCZxZDL" + } + }, + "elastic_agent": { + "id": "df58bff0-5714-11eb-b094-915beebb3c66", + "snapshot": true, + "version": "7.11.0" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | +| stan.cluster.id | The cluster ID | keyword | +| stan.server.id | The server ID | keyword | +| stan.stats.bytes | Number of bytes consumed across all STAN queues | long | +| stan.stats.channels | The number of STAN channels | integer | +| stan.stats.clients | The number of STAN clients | integer | +| stan.stats.messages | Number of messages across all STAN queues | long | +| stan.stats.role | If clustered, role of this node in the cluster (Leader, Follower, Candidate) | keyword | +| stan.stats.state | The cluster / streaming configuration state (STANDALONE, CLUSTERED) | keyword | +| stan.stats.subscriptions | The number of STAN streaming subscriptions | integer | + + +### channels + +This is the `channels` dataset of the STAN package, in charge of retrieving +metrics about channels from a STAN instance. + +An example event for `channels` looks as following: + +```$json +{ + "@timestamp": "2021-01-15T12:23:32.592Z", + "service": { + "address": "http://elastic-package-service_stan_1:8222/streaming/channelsz?subs=1", + "type": "stan" + }, + "event": { + "duration": 8406132380, + "dataset": "stan.channels", + "module": "stan" + }, + "metricset": { + "name": "channels", + "period": 60000 + }, + "stan": { + "cluster": { + "id": "test-cluster" + }, + "server": { + "id": "kvQEpbFak88fHAnWCZxZDL" + }, + "channels": { + "depth": 3966, + "name": "bar", + "messages": 4990, + "bytes": 5214423, + "first_seq": 1, + "last_seq": 4990 + } + }, + "elastic_agent": { + "version": "7.11.0", + "id": "df58bff0-5714-11eb-b094-915beebb3c66", + "snapshot": true + }, + "ecs": { + "version": "1.7.0" + }, + "data_stream": { + "type": "metrics", + "dataset": "stan.channels", + "namespace": "default" + }, + "host": { + "architecture": "x86_64", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "88c3c3ec3afebed7631b44a69754359e", + "name": "ec072aa02d8b", + "containerized": true, + "ip": [ + "172.19.0.5" + ], + "mac": [ + "02:42:ac:13:00:05" + ], + "hostname": "ec072aa02d8b" + }, + "agent": { + "version": "7.11.0", + "hostname": "ec072aa02d8b", + "ephemeral_id": "8d73aff0-201b-4260-9e89-cd519348de03", + "id": "67b9c377-7d0c-4a69-9351-2befe6386fbd", + "name": "ec072aa02d8b", + "type": "metricbeat" + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | +| stan.channels.bytes | The number of STAN bytes in the channel | long | +| stan.channels.depth | Queue depth based upon current sequence number and highest reported subscriber sequence number | long | +| stan.channels.first_seq | First sequence number stored in the channel. If first_seq > min([seq in subscriptions]) data loss has possibly occurred | long | +| stan.channels.last_seq | Last sequence number stored in the channel | long | +| stan.channels.messages | The number of STAN streaming messages | long | +| stan.channels.name | The name of the STAN streaming channel | keyword | +| stan.cluster.id | The cluster ID | keyword | +| stan.server.id | The server ID | keyword | + + +### subscriptions + +This is the `subscriptions` dataset of the STAN package, in charge of retrieving +metrics about subscriptions from a STAN instance. + +An example event for `subscriptions` looks as following: + +```$json +{ + "@timestamp": "2021-01-15T12:25:32.509Z", + "ecs": { + "version": "1.7.0" + }, + "agent": { + "ephemeral_id": "8d73aff0-201b-4260-9e89-cd519348de03", + "id": "67b9c377-7d0c-4a69-9351-2befe6386fbd", + "name": "ec072aa02d8b", + "type": "metricbeat", + "version": "7.11.0", + "hostname": "ec072aa02d8b" + }, + "metricset": { + "name": "subscriptions", + "period": 60000 + }, + "data_stream": { + "type": "metrics", + "dataset": "stan.subscriptions", + "namespace": "default" + }, + "elastic_agent": { + "version": "7.11.0", + "id": "df58bff0-5714-11eb-b094-915beebb3c66", + "snapshot": true + }, + "host": { + "architecture": "x86_64", + "name": "ec072aa02d8b", + "os": { + "kernel": "4.9.184-linuxkit", + "codename": "Core", + "platform": "centos", + "version": "7 (Core)", + "family": "redhat", + "name": "CentOS Linux" + }, + "id": "88c3c3ec3afebed7631b44a69754359e", + "containerized": true, + "ip": [ + "172.19.0.5" + ], + "mac": [ + "02:42:ac:13:00:05" + ], + "hostname": "ec072aa02d8b" + }, + "event": { + "dataset": "stan.subscriptions", + "module": "stan", + "duration": 935334325 + }, + "service": { + "address": "http://elastic-package-service_stan_1:8222/streaming/channelsz?subs=1", + "type": "stan" + }, + "stan": { + "subscriptions": { + "stalled": true, + "pending": 1024, + "id": "benchmark-sub-1", + "channel": "bar", + "last_sent": 1024, + "offline": true + }, + "server": { + "id": "kvQEpbFak88fHAnWCZxZDL" + }, + "cluster": { + "id": "test-cluster" + } + } +} +``` + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version | keyword | +| service.address | Service address | keyword | +| service.type | Service type | keyword | +| stan.cluster.id | The cluster ID | keyword | +| stan.server.id | The server ID | keyword | +| stan.subscriptions.channel | The name of the STAN channel the subscription is associated with | keyword | +| stan.subscriptions.id | The name of the STAN channel subscription (client_id) | keyword | +| stan.subscriptions.last_sent | Last known sequence number of the subscription that was acked | long | +| stan.subscriptions.offline | Is the subscriber marked as offline? | boolean | +| stan.subscriptions.pending | Number of pending messages from / to the subscriber | long | +| stan.subscriptions.queue | The name of the NATS queue that the STAN channel subscription is associated with, if any | keyword | +| stan.subscriptions.stalled | Is the subscriber known to be stalled? | boolean | diff --git a/packages/stan/img/logs-stan-overview.png b/packages/stan/img/logs-stan-overview.png new file mode 100644 index 00000000000..7d0cca6cb2c Binary files /dev/null and b/packages/stan/img/logs-stan-overview.png differ diff --git a/packages/stan/img/metrics-stan-overview.png b/packages/stan/img/metrics-stan-overview.png new file mode 100644 index 00000000000..a6ed419fe88 Binary files /dev/null and b/packages/stan/img/metrics-stan-overview.png differ diff --git a/packages/stan/img/stan.svg b/packages/stan/img/stan.svg new file mode 100644 index 00000000000..5a1d6e9a52f --- /dev/null +++ b/packages/stan/img/stan.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/stan/kibana/dashboard/stan-Logs-stan-overview.json b/packages/stan/kibana/dashboard/stan-Logs-stan-overview.json new file mode 100644 index 00000000000..7854176b68b --- /dev/null +++ b/packages/stan/kibana/dashboard/stan-Logs-stan-overview.json @@ -0,0 +1,245 @@ +{ + "attributes": { + "description": "Overview of STAN server statistics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "optionsJSON": { + "darkTheme": false, + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Message Types Timeline" + }, + "gridData": { + "h": 11, + "i": "1", + "w": 17, + "x": 0, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "title": "Message Types Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Communication Directions" + }, + "gridData": { + "h": 11, + "i": "2", + "w": 17, + "x": 31, + "y": 0 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "title": "Communication Directions", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Topics Timeline" + }, + "gridData": { + "h": 12, + "i": "3", + "w": 25, + "x": 0, + "y": 20 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "title": "Topics Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Bytes Timeline", + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 9, + "i": "4", + "w": 12, + "x": 11, + "y": 11 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "title": "Bytes Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Communication Directions Distribution", + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 9, + "i": "5", + "w": 11, + "x": 0, + "y": 11 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "title": "Communication Directions Distribution", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Log Level Distribution", + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 9, + "i": "6", + "w": 11, + "x": 37, + "y": 11 + }, + "panelIndex": "6", + "panelRefName": "panel_5", + "title": "Log Level Distribution", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Message Type Distribution", + "vis": { + "legendOpen": false + } + }, + "gridData": { + "h": 11, + "i": "7", + "w": 14, + "x": 17, + "y": 0 + }, + "panelIndex": "7", + "panelRefName": "panel_6", + "title": "Message Type Distribution", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Log Level Timeline" + }, + "gridData": { + "h": 9, + "i": "8", + "w": 14, + "x": 23, + "y": 11 + }, + "panelIndex": "8", + "panelRefName": "panel_7", + "title": "Log Level Timeline", + "version": "7.10.0" + }, + { + "embeddableConfig": { + "hidePanelTitles": false, + "title": "Client IP Count Timeline" + }, + "gridData": { + "h": 12, + "i": "9", + "w": 22, + "x": 25, + "y": 20 + }, + "panelIndex": "9", + "panelRefName": "panel_8", + "title": "Client IP Count Timeline", + "version": "7.10.0" + } + ], + "timeRestore": false, + "title": "[Logs STAN] Overview", + "version": 1 + }, + "id": "stan-Logs-stan-overview", + "migrationVersion": { + "dashboard": "7.9.3" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "stan-6987a800-41a8-11e9-a4da-b1df688edbcd", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "stan-0b2061d0-41ad-11e9-a4da-b1df688edbcd", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "stan-4a6d9ec0-41a8-11e9-a4da-b1df688edbcd", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "stan-c3d1ab80-41a8-11e9-a4da-b1df688edbcd", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "stan-7716c780-41ad-11e9-a4da-b1df688edbcd", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "stan-3f6cca40-41ae-11e9-a4da-b1df688edbcd", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "stan-7ed62870-41ae-11e9-a4da-b1df688edbcd", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "stan-04083600-41af-11e9-a4da-b1df688edbcd", + "name": "panel_7", + "type": "visualization" + }, + { + "id": "stan-c669ae20-41ed-11e9-ac5c-71ffa38a62e3", + "name": "panel_8", + "type": "visualization" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/stan/kibana/dashboard/stan-b6a60340-4371-11ea-b0c6-cb14c0977bd1.json b/packages/stan/kibana/dashboard/stan-b6a60340-4371-11ea-b0c6-cb14c0977bd1.json new file mode 100644 index 00000000000..d0bdcff84df --- /dev/null +++ b/packages/stan/kibana/dashboard/stan-b6a60340-4371-11ea-b0c6-cb14c0977bd1.json @@ -0,0 +1,74 @@ +{ + "attributes": { + "description": "Metrics for STAN channels / subjects", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "98c9a793-903a-47af-900b-8becd2493d0a", + "w": 24, + "x": 24, + "y": 0 + }, + "panelIndex": "98c9a793-903a-47af-900b-8becd2493d0a", + "panelRefName": "panel_0", + "version": "7.5.2" + }, + { + "embeddableConfig": { + "enhancements": {} + }, + "gridData": { + "h": 15, + "i": "d11727cf-8d05-45ae-9ae0-2f3b79ab7eda", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "d11727cf-8d05-45ae-9ae0-2f3b79ab7eda", + "panelRefName": "panel_1", + "version": "7.5.2" + } + ], + "timeRestore": false, + "title": "[Metrics Stan] Channel Overview Metrics", + "version": 1 + }, + "id": "stan-b6a60340-4371-11ea-b0c6-cb14c0977bd1", + "migrationVersion": { + "dashboard": "7.11.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "stan-46a07ac0-436d-11ea-b0c6-cb14c0977bd1", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "stan-0e412fe0-4371-11ea-b0c6-cb14c0977bd1", + "name": "panel_1", + "type": "visualization" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-04083600-41af-11e9-a4da-b1df688edbcd.json b/packages/stan/kibana/visualization/stan-04083600-41af-11e9-a4da-b1df688edbcd.json new file mode 100644 index 00000000000..3eb11f23a3b --- /dev/null +++ b/packages/stan/kibana/visualization/stan-04083600-41af-11e9-a4da-b1df688edbcd.json @@ -0,0 +1,131 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Log Level Timeline [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "log.level", + "size": 10 + }, + "schema": "group", + "type": "significant_terms" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "interpolate": "linear", + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "area", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "area", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ] + }, + "title": "Log Level Timeline [Logs STAN]", + "type": "area" + } + }, + "id": "stan-04083600-41af-11e9-a4da-b1df688edbcd", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-0b2061d0-41ad-11e9-a4da-b1df688edbcd.json b/packages/stan/kibana/visualization/stan-0b2061d0-41ad-11e9-a4da-b1df688edbcd.json new file mode 100644 index 00000000000..921ad9ef6e8 --- /dev/null +++ b/packages/stan/kibana/visualization/stan-0b2061d0-41ad-11e9-a4da-b1df688edbcd.json @@ -0,0 +1,130 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Communication Directions [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "network.direction", + "size": 2 + }, + "schema": "group", + "type": "significant_terms" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "normal", + "show": "true", + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ] + }, + "title": "Communication Directions [Logs STAN]", + "type": "line" + } + }, + "id": "stan-0b2061d0-41ad-11e9-a4da-b1df688edbcd", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-0e412fe0-4371-11ea-b0c6-cb14c0977bd1.json b/packages/stan/kibana/visualization/stan-0e412fe0-4371-11ea-b0c6-cb14c0977bd1.json new file mode 100644 index 00000000000..23d6c642c80 --- /dev/null +++ b/packages/stan/kibana/visualization/stan-0e412fe0-4371-11ea-b0c6-cb14c0977bd1.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "Queue depth of STAN channels, summed per channel", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Channel Queue Depth [Metrics Stan]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metrics-*", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "bar", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "Queue Depth", + "line_width": 1, + "metrics": [ + { + "field": "stan.channels.depth", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "max" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "stan.channels.name", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "type": "timeseries" + }, + "title": "Channel Queue Depth [Metrics Stan]", + "type": "metrics" + } + }, + "id": "stan-0e412fe0-4371-11ea-b0c6-cb14c0977bd1", + "migrationVersion": { + "visualization": "7.11.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-3f6cca40-41ae-11e9-a4da-b1df688edbcd.json b/packages/stan/kibana/visualization/stan-3f6cca40-41ae-11e9-a4da-b1df688edbcd.json new file mode 100644 index 00000000000..e36918518f9 --- /dev/null +++ b/packages/stan/kibana/visualization/stan-3f6cca40-41ae-11e9-a4da-b1df688edbcd.json @@ -0,0 +1,69 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Log Level Distribution [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "log.level", + "size": 10 + }, + "schema": "segment", + "type": "significant_terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": true, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Log Level Distribution [Logs STAN]", + "type": "pie" + } + }, + "id": "stan-3f6cca40-41ae-11e9-a4da-b1df688edbcd", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-46a07ac0-436d-11ea-b0c6-cb14c0977bd1.json b/packages/stan/kibana/visualization/stan-46a07ac0-436d-11ea-b0c6-cb14c0977bd1.json new file mode 100644 index 00000000000..656d6548d64 --- /dev/null +++ b/packages/stan/kibana/visualization/stan-46a07ac0-436d-11ea-b0c6-cb14c0977bd1.json @@ -0,0 +1,66 @@ +{ + "attributes": { + "description": "Number of messages in each channel / subject", + "kibanaSavedObjectMeta": { + "searchSourceJSON": {} + }, + "title": "Channel Messages [Metrics Stan]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metrics-*", + "default_timefield": "@timestamp", + "id": "61ca57f0-469d-11e7-af02-69e470af7417", + "index_pattern": "metrics-*", + "interval": "", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "bar", + "color": "#68BC00", + "fill": 0.5, + "formatter": "number", + "id": "61ca57f1-469d-11e7-af02-69e470af7417", + "label": "STAN Queue Messages", + "line_width": 1, + "metrics": [ + { + "field": "stan.channels.messages", + "id": "61ca57f2-469d-11e7-af02-69e470af7417", + "type": "max" + } + ], + "point_size": 1, + "separate_axis": 0, + "split_color_mode": "gradient", + "split_mode": "terms", + "stacked": "none", + "terms_field": "stan.channels.name", + "type": "timeseries" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "", + "type": "timeseries" + }, + "title": "Channel Messages [Metrics Stan]", + "type": "metrics" + } + }, + "id": "stan-46a07ac0-436d-11ea-b0c6-cb14c0977bd1", + "migrationVersion": { + "visualization": "7.11.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-4a6d9ec0-41a8-11e9-a4da-b1df688edbcd.json b/packages/stan/kibana/visualization/stan-4a6d9ec0-41a8-11e9-a4da-b1df688edbcd.json new file mode 100644 index 00000000000..f575473609e --- /dev/null +++ b/packages/stan/kibana/visualization/stan-4a6d9ec0-41a8-11e9-a4da-b1df688edbcd.json @@ -0,0 +1,130 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Topics Timeline [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "stan.log.msg.subject", + "size": 5 + }, + "schema": "group", + "type": "significant_terms" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "normal", + "show": "true", + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ] + }, + "title": "Topics Timeline [Logs STAN]", + "type": "line" + } + }, + "id": "stan-4a6d9ec0-41a8-11e9-a4da-b1df688edbcd", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-6987a800-41a8-11e9-a4da-b1df688edbcd.json b/packages/stan/kibana/visualization/stan-6987a800-41a8-11e9-a4da-b1df688edbcd.json new file mode 100644 index 00000000000..a10eeed3fb5 --- /dev/null +++ b/packages/stan/kibana/visualization/stan-6987a800-41a8-11e9-a4da-b1df688edbcd.json @@ -0,0 +1,133 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Message Types Timeline [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "stan.log.msg.type", + "size": 15 + }, + "schema": "group", + "type": "significant_terms" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "interpolate": "cardinal", + "mode": "normal", + "show": "true", + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ] + }, + "title": "Message Types Timeline [Logs STAN]", + "type": "line" + } + }, + "id": "stan-6987a800-41a8-11e9-a4da-b1df688edbcd", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-7716c780-41ad-11e9-a4da-b1df688edbcd.json b/packages/stan/kibana/visualization/stan-7716c780-41ad-11e9-a4da-b1df688edbcd.json new file mode 100644 index 00000000000..683a14db028 --- /dev/null +++ b/packages/stan/kibana/visualization/stan-7716c780-41ad-11e9-a4da-b1df688edbcd.json @@ -0,0 +1,69 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Communication Directions Distribution [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "network.direction", + "size": 2 + }, + "schema": "segment", + "type": "significant_terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": true, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Communication Directions Distribution [Logs STAN]", + "type": "pie" + } + }, + "id": "stan-7716c780-41ad-11e9-a4da-b1df688edbcd", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-7ed62870-41ae-11e9-a4da-b1df688edbcd.json b/packages/stan/kibana/visualization/stan-7ed62870-41ae-11e9-a4da-b1df688edbcd.json new file mode 100644 index 00000000000..d3a54c015c7 --- /dev/null +++ b/packages/stan/kibana/visualization/stan-7ed62870-41ae-11e9-a4da-b1df688edbcd.json @@ -0,0 +1,69 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Message Type Distribution [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "stan.log.msg.type", + "size": 15 + }, + "schema": "segment", + "type": "significant_terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": true, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Message Type Distribution [Logs STAN]", + "type": "pie" + } + }, + "id": "stan-7ed62870-41ae-11e9-a4da-b1df688edbcd", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-c3d1ab80-41a8-11e9-a4da-b1df688edbcd.json b/packages/stan/kibana/visualization/stan-c3d1ab80-41a8-11e9-a4da-b1df688edbcd.json new file mode 100644 index 00000000000..6b696a1039f --- /dev/null +++ b/packages/stan/kibana/visualization/stan-c3d1ab80-41a8-11e9-a4da-b1df688edbcd.json @@ -0,0 +1,123 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Bytes Timeline [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Sum of Message Bytes", + "field": "stan.log.msg.bytes" + }, + "schema": "metric", + "type": "sum" + }, + { + "enabled": true, + "id": "2", + "schema": "segment", + "type": "date_histogram" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Sum of Message Bytes" + }, + "drawLinesBetweenPoints": true, + "mode": "normal", + "show": "true", + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Sum of Message Bytes" + }, + "type": "value" + } + ] + }, + "title": "Bytes Timeline [Logs STAN]", + "type": "line" + } + }, + "id": "stan-c3d1ab80-41a8-11e9-a4da-b1df688edbcd", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/kibana/visualization/stan-c669ae20-41ed-11e9-ac5c-71ffa38a62e3.json b/packages/stan/kibana/visualization/stan-c669ae20-41ed-11e9-ac5c-71ffa38a62e3.json new file mode 100644 index 00000000000..c3485a2b3ab --- /dev/null +++ b/packages/stan/kibana/visualization/stan-c669ae20-41ed-11e9-ac5c-71ffa38a62e3.json @@ -0,0 +1,150 @@ +{ + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "data_stream.dataset: stan.log" + } + } + }, + "title": "Client IP Count Timeline [Logs STAN]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "client.ip", + "ipRangeType": "fromTo", + "ranges": { + "fromTo": [ + { + "from": "0.0.0.0", + "to": "127.255.255.255" + }, + { + "from": "128.0.0.0", + "to": "191.255.255.255" + } + ], + "mask": [ + { + "mask": "0.0.0.0/1" + }, + { + "mask": "128.0.0.0/2" + } + ] + } + }, + "schema": "group", + "type": "ip_range" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "normal", + "show": "true", + "showCircles": true, + "type": "line", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "line", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ] + }, + "title": "Client IP Count Timeline [Logs STAN]", + "type": "line" + } + }, + "id": "stan-c669ae20-41ed-11e9-ac5c-71ffa38a62e3", + "migrationVersion": { + "visualization": "7.10.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "logs-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization" +} \ No newline at end of file diff --git a/packages/stan/manifest.yml b/packages/stan/manifest.yml new file mode 100644 index 00000000000..ca63df759a5 --- /dev/null +++ b/packages/stan/manifest.yml @@ -0,0 +1,48 @@ +name: stan +title: STAN +version: 0.1.0 +release: beta +description: STAN Integration +type: integration +icons: + - src: /img/stan.svg + title: STAN Logo + size: 32x32 + type: image/svg+xml +format_version: 1.0.0 +license: basic +categories: + - message_queue +conditions: + kibana.version: ">=7.11.0" +screenshots: + - src: /img/metrics-stan-overview.png + title: Metrics STAN Dashboard + size: 1829x447 + type: image/png + - src: /img/logs-stan-overview.png + title: Logs STAN Dashboard + size: 1829x447 + type: image/png +policy_templates: + - name: stan + title: STAN Logs and Metrics + description: Collect logs and metrics from STAN instances + inputs: + - type: logfile + title: 'Collect STAN application logs (input: logfile)' + description: 'Collecting application logs from STAN instances (input: logfile)' + - type: stan/metrics + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + default: + - localhost:8222 + title: Collect STAN metrics + description: Collecting metrics about channels, stats and subscriptions from STAN instances +owner: + github: elastic/integrations