Add stan package

packages/stan/_dev/build/docs/README.md

@@ -0,0 +1,50 @@
+# STAN integration
+
+This integration is used to collect logs and metrics from [STAN servers](https://github.com/nats-io/stan.go).
+The integration collects metrics from [STAN monitoring server APIs](https://github.com/nats-io/nats-streaming-server/blob/master/server/monitor.go).
+
+
+## Compatibility
+
+The STAN package is tested with Stan 0.15.1
+
+## Logs
+
+### log
+
+The `log` dataset collects the STAN logs.
+
+{{event "log"}}
+
+{{fields "log"}}
+
+## Metrics
+
+The default datasets are `stats`, `channels`, and `subscriptions`.
+
+### stats
+
+This is the `stats` dataset of the STAN package, in charge of retrieving generic
+metrics from a STAN instance.
+
+{{event "stats"}}
+
+{{fields "stats"}}
+
+### channels
+
+This is the `channels` dataset of the STAN package, in charge of retrieving
+metrics about channels from a STAN instance.
+
+{{event "channels"}}
+
+{{fields "channels"}}
+
+### subscriptions
+
+This is the `subscriptions` dataset of the STAN package, in charge of retrieving
+metrics about subscriptions from a STAN instance.
+
+{{event "subscriptions"}}
+
+{{fields "subscriptions"}}

packages/stan/_dev/deploy/docker/Dockerfile

@@ -0,0 +1,26 @@
+ARG STAN_VERSION=0.15.1
+FROM nats-streaming:$STAN_VERSION
+
+# build stage
+FROM golang:1.13-alpine3.11 AS build-env
+RUN apk --no-cache add build-base git mercurial gcc
+RUN cd src && go get -d github.com/nats-io/stan.go/
+RUN cd src/github.com/nats-io/stan.go/examples/stan-bench && git checkout tags/v0.5.2 && go build .
+
+# create an enhanced container with nc command available since nats is based
+# on scratch image making healthcheck impossible
+FROM alpine:latest
+RUN apk add --no-cache --upgrade bash
+COPY --from=0 nats-streaming-server /nats-streaming-server
+COPY --from=build-env /go/src/github.com/nats-io/stan.go/examples/stan-bench/stan-bench /stan-bench
+# Expose client, management, and cluster ports
+EXPOSE 4222 8222
+ADD healthcheck.sh /healthcheck.sh
+RUN ["chmod", "+x", "/healthcheck.sh"]
+ADD run.sh /run.sh
+
+# Healthcheck waits until channels have been created by the benchmark that runs inside
+HEALTHCHECK --interval=1s --retries=100 CMD /healthcheck.sh
+
+# Run with default memory based store
+ENTRYPOINT ["/run.sh"]

packages/stan/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '2.3'
+services:
+  stan:
+    # Commented out `image:` below until we have a process to refresh the hosted images from
+    # Dockerfiles in this repo. Until then, we build the image locally using `build:` below.
+    # image: docker.elastic.co/integrations-ci/beats-stan:${SERVICE_VERSION:-2.4.20}-1
+    build: .
+    ports:
+      - 8222
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/var/log/stan

packages/stan/_dev/deploy/docker/healthcheck.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+res=$(wget -q -O - http://0.0.0.0:8222/streaming/channelsz | sed -n 's/"count": \([[:digit:]]\+\),/\1/p')
+
+if [[ $res -gt 0 ]]; then
+	exit 0
+fi
+
+exit 1

packages/stan/_dev/deploy/docker/run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+/nats-streaming-server -DV -l /var/log/stan/stan.log -m 8222 &
+sleep 2
+while true; do /stan-bench -np 0 -ns 100 -qgroup T -n 100000000 -ms 1024 foo; done
+#while true; do /stan-bench -np 10 -ns 10 -n 1000000000 -ms 1024 bar; done &
+
+# Make sure the container keeps running
+tail -f /dev/null
+

packages/stan/_dev/deploy/variants.yml

@@ -0,0 +1,4 @@
+variants:
+  v1:
+    SERVICE_VERSION: 0.15.1
+default: v1

packages/stan/data_stream/channels/_dev/test/system/test-default-config.yml

@@ -0,0 +1,5 @@
+vars:
+  hosts:
+    - http://{{Hostname}}:{{Port}}
+data_stream:
+  vars: ~

packages/stan/data_stream/channels/agent/stream/stream.yml.hbs

@@ -0,0 +1,6 @@
+metricsets: ["channels"]
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+period: {{period}}

packages/stan/data_stream/channels/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

packages/stan/data_stream/channels/fields/ecs.yml

@@ -0,0 +1,9 @@
+- name: ecs.version
+  type: keyword
+  description: ECS version
+- name: service.address
+  type: keyword
+  description: Service address
+- name: service.type
+  type: keyword
+  description: Service type

packages/stan/data_stream/channels/fields/fields.yml

@@ -0,0 +1,28 @@
+- name: stan.channels
+  type: group
+  release: ga
+  fields:
+    - name: name
+      type: keyword
+      description: |
+        The name of the STAN streaming channel
+    - name: messages
+      type: long
+      description: |
+        The number of STAN streaming messages
+    - name: bytes
+      type: long
+      description: |
+        The number of STAN bytes in the channel
+    - name: first_seq
+      type: long
+      description: |
+        First sequence number stored in the channel. If first_seq > min([seq in subscriptions]) data loss has possibly occurred
+    - name: last_seq
+      type: long
+      description: |
+        Last sequence number stored in the channel
+    - name: depth
+      type: long
+      description: |
+        Queue depth based upon current sequence number and highest reported subscriber sequence number

packages/stan/data_stream/channels/fields/package-fields.yml

@@ -0,0 +1,11 @@
+- name: stan
+  type: group
+  fields:
+    - name: server.id
+      type: keyword
+      description: |
+        The server ID
+    - name: cluster.id
+      type: keyword
+      description: |
+        The cluster ID

packages/stan/data_stream/channels/manifest.yml

@@ -0,0 +1,15 @@
+type: metrics
+title: Stan channels metrics
+release: experimental
+streams:
+  - input: stan/metrics
+    vars:
+      - name: period
+        type: text
+        title: Period
+        multi: false
+        required: true
+        show_user: true
+        default: 60s
+    title: Stan channels metrics
+    description: Collect Stan channels metrics

packages/stan/data_stream/channels/sample_event.json

@@ -0,0 +1,74 @@
+{
+    "@timestamp": "2021-01-15T12:23:32.592Z",
+    "service": {
+        "address": "http://elastic-package-service_stan_1:8222/streaming/channelsz?subs=1",
+        "type": "stan"
+    },
+    "event": {
+        "duration": 8406132380,
+        "dataset": "stan.channels",
+        "module": "stan"
+    },
+    "metricset": {
+        "name": "channels",
+        "period": 60000
+    },
+    "stan": {
+        "cluster": {
+            "id": "test-cluster"
+        },
+        "server": {
+            "id": "kvQEpbFak88fHAnWCZxZDL"
+        },
+        "channels": {
+            "depth": 3966,
+            "name": "bar",
+            "messages": 4990,
+            "bytes": 5214423,
+            "first_seq": 1,
+            "last_seq": 4990
+        }
+    },
+    "elastic_agent": {
+        "version": "7.11.0",
+        "id": "df58bff0-5714-11eb-b094-915beebb3c66",
+        "snapshot": true
+    },
+    "ecs": {
+        "version": "1.7.0"
+    },
+    "data_stream": {
+        "type": "metrics",
+        "dataset": "stan.channels",
+        "namespace": "default"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "os": {
+            "kernel": "4.9.184-linuxkit",
+            "codename": "Core",
+            "platform": "centos",
+            "version": "7 (Core)",
+            "family": "redhat",
+            "name": "CentOS Linux"
+        },
+        "id": "88c3c3ec3afebed7631b44a69754359e",
+        "name": "ec072aa02d8b",
+        "containerized": true,
+        "ip": [
+            "172.19.0.5"
+        ],
+        "mac": [
+            "02:42:ac:13:00:05"
+        ],
+        "hostname": "ec072aa02d8b"
+    },
+    "agent": {
+        "version": "7.11.0",
+        "hostname": "ec072aa02d8b",
+        "ephemeral_id": "8d73aff0-201b-4260-9e89-cd519348de03",
+        "id": "67b9c377-7d0c-4a69-9351-2befe6386fbd",
+        "name": "ec072aa02d8b",
+        "type": "metricbeat"
+    }
+}

packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log

@@ -0,0 +1,69 @@
+[7] 2021/01/13 14.20:06.981022 [INF] STREAM: Starting nats-streaming-server[test-cluster] version 0.15.1
+[7] 2021/01/13 14.20:06.981055 [INF] STREAM: ServerID: tnudw5OA2pv04Sn0JpebGy
+[7] 2021/01/13 14.20:06.981069 [INF] STREAM: Go version: go1.11.10
+[7] 2021/01/13 14.20:06.981082 [INF] STREAM: Git commit: [8e6aa7c]
+[7] 2021/01/13 14.20:06.981951 [INF] Starting nats-server version 2.0.0
+[7] 2021/01/13 14.20:06.981983 [DBG] Go build version go1.11.10
+[7] 2021/01/13 14.20:06.981996 [INF] Git commit [not set]
+[7] 2021/01/13 14.20:06.982082 [INF] Starting http monitor on 0.0.0.0:8222
+[7] 2021/01/13 14.20:06.982116 [INF] Listening for client connections on 0.0.0.0:4222
+[7] 2021/01/13 14.20:06.982127 [INF] Server id is NCE3NFAMOIJHIR6KK53KBPN6C44LXZ6G4ULGLIWIBLYOLG5Z5LVCQSOB
+[7] 2021/01/13 14.20:06.982136 [INF] Server is ready
+[7] 2021/01/13 14.20:06.982149 [DBG] Get non local IPs for "0.0.0.0"
+[7] 2021/01/13 14.20:06.982402 [DBG]  ip=192.168.240.2
+[7] 2021/01/13 14.20:07.008167 [DBG] 127.0.0.1:33150 - cid:1 - Client connection created
+[7] 2021/01/13 14.20:07.009385 [DBG] 127.0.0.1:33152 - cid:2 - Client connection created
+[7] 2021/01/13 14.20:07.010819 [DBG] 127.0.0.1:33154 - cid:3 - Client connection created
+[7] 2021/01/13 14.20:07.011557 [INF] STREAM: Recovering the state...
+[7] 2021/01/13 14.20:07.011583 [INF] STREAM: No recovered state
+[7] 2021/01/13 14.20:07.263504 [INF] STREAM: Message store is MEMORY
+[7] 2021/01/13 14.20:07.263623 [INF] STREAM: ---------- Store Limits ----------
+[7] 2021/01/13 14.20:07.263653 [INF] STREAM: Channels:                  100 *
+[7] 2021/01/13 14.20:07.263671 [INF] STREAM: --------- Channels Limits --------
+[7] 2021/01/13 14.20:07.263690 [INF] STREAM:   Subscriptions:          1000 *
+[7] 2021/01/13 14.20:07.263710 [INF] STREAM:   Messages     :       1000000 *
+[7] 2021/01/13 14.20:07.263728 [INF] STREAM:   Bytes        :     976.56 MB *
+[7] 2021/01/13 14.20:07.263749 [INF] STREAM:   Age          :     unlimited *
+[7] 2021/01/13 14.20:07.263767 [INF] STREAM:   Inactivity   :     unlimited *
+[7] 2021/01/13 14.20:07.263788 [INF] STREAM: ----------------------------------
+[7] 2021/01/13 14.20:08.988091 [DBG] 127.0.0.1:33160 - cid:4 - Client connection created
+[7] 2021/01/13 14.20:09.010952 [INF] STREAM: Channel "bar" has been created
+[7] 2021/01/13 14:22:50.499525 [TRC] 127.0.0.1:60748 - cid:3 - <<- [SUB _INBOX.mUz7h6B3wGNf7P7bhyQldF  43]
+[7] 2021/01/13 14:22:50.497650 [DBG] 127.0.0.1:60800 - cid:27 - Client connection closed
+[7] 2021/01/13 14:22:50.499560 [DBG] 127.0.0.1:60812 - cid:34 - Client connection closed
+[7] 2021/01/13 14:22:50.499565 [DBG] 127.0.0.1:60818 - cid:36 - Client connection closed
+[7] 2021/01/13 14:22:50.499560 [DBG] 127.0.0.1:60816 - cid:35 - Client connection closed
+[7] 2021/01/13 14:22:50.499586 [TRC] 127.0.0.1:60748 - cid:3 - <<- [PING]
+[7] 2021/01/13 14:22:50.499599 [DBG] 127.0.0.1:60820 - cid:38 - Client connection closed
+[7] 2021/01/13 14:22:50.499607 [TRC] 127.0.0.1:60748 - cid:3 - ->> [PONG]
+[7] 2021/01/13 14:22:50.499634 [DBG] 127.0.0.1:60804 - cid:30 - Client connection closed
+[7] 2021/01/13 14:22:50.499651 [DBG] 127.0.0.1:60806 - cid:32 - Client connection closed
+[7] 2021/01/13 14:22:50.499656 [DBG] 127.0.0.1:60814 - cid:42 - Client connection closed
+[7] 2021/01/13 14:22:50.499694 [DBG] 127.0.0.1:60808 - cid:33 - Client connection closed
+[7] 2021/01/13 14:22:50.499717 [DBG] 127.0.0.1:60802 - cid:31 - Client connection closed
+[7] 2021/01/13 14:22:50.499727 [TRC] 127.0.0.1:60748 - cid:3 - <<- [SUB _INBOX.mUz7h6B3wGNf7P7bhyQlf6  44]
+[7] 2021/01/13 14:22:50.499769 [TRC] 127.0.0.1:60748 - cid:3 - <<- [PING]
+[7] 2021/01/13 14:22:50.488515 [TRC] 127.0.0.1:60910 - cid:83 - ->> [MSG _INBOX.ZXYA2FSF5VwytqOUdLsWfp.RdKmEViH 3 31]
+[7] 2021/01/13 14:22:50.499782 [TRC] 127.0.0.1:60748 - cid:3 - ->> [PONG]
+[7] 2021/01/13 14:22:50.499811 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsXOL.kiHXi6ux 31]
+[7] 2021/01/13 14:22:50.499850 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQlS7"]
+[7] 2021/01/13 14:22:50.499850 [DBG] 127.0.0.1:60910 - cid:83 - Client connection closed
+[7] 2021/01/13 14:22:50.499961 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsUbq.CeeDx8Zh 31]
+[7] 2021/01/13 14:22:50.500004 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQlTy"]
+[7] 2021/01/13 14:22:50.500142 [DBG] 127.0.0.1:60810 - cid:29 - Client connection closed
+[7] 2021/01/13 14:22:50.508653 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsUsX.7vrrWV3v 31]
+[7] 2021/01/13 14:22:50.508725 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQm1M"]
+[7] 2021/01/13 14:22:50.508872 [TRC] 127.0.0.1:60748 - cid:3 - <<- [SUB _INBOX.mUz7h6B3wGNf7P7bhyQm3D  57]
+[7] 2021/01/13 14:22:50.509016 [TRC] 127.0.0.1:60748 - cid:3 - <<- [PING]
+[7] 2021/01/13 14:22:50.509046 [TRC] 127.0.0.1:60748 - cid:3 - ->> [PONG]
+[7] 2021/01/13 14:22:50.509787 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsYyH.3U691ECJ 31]
+[7] 2021/01/13 14:22:50.509825 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQm3D"]
+[7] 2021/01/13 14:22:50.509825 [TRC] 127.0.0.1:60748 - cid:3 - <<- [SUB _INBOX.mUz7h6B3wGNf7P7bhyQm54  58]
+[7] 2021/01/13 14:22:50.509899 [TRC] 127.0.0.1:60748 - cid:3 - <<- [PING]
+[7] 2021/01/13 14:22:50.509912 [TRC] 127.0.0.1:60748 - cid:3 - ->> [PONG]
+[7] 2021/01/13 14:22:50.510036 [TRC] 127.0.0.1:60744 - cid:1 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsYpN.R6hUgpF6 31]
+[7] 2021/01/13 14:22:50.510073 [TRC] 127.0.0.1:60744 - cid:1 - <<- MSG_PAYLOAD: ["\x12\x1d_INBOX.mUz7h6B3wGNf7P7bhyQm54"]
+[7] 2021/01/13 14:22:50.949500 [TRC] 127.0.0.1:60746 - cid:2 - <<- [PUB _INBOX.ZXYA2FSF5VwytqOUdLsWF7.IqlBSvhV 220]
+[7] 2021/01/13 14:22:50.949675 [TRC] 127.0.0.1:60746 - cid:2 - <<- MSG_PAYLOAD: ["\n _STAN.pub.mUz7h6B3wGNf7P7bhyQkHm\x12 _STAN.sub.mUz7h6B3wGNf7P7bhyQkHm\x1a\"_STAN.unsub.mUz7h6B3wGNf7P7bhyQkHm\"\"_STAN.close.mUz7h6B3wGNf7P7bhyQkHm2%_STAN.subclose.mUz7h6B3wGNf7P7bhyQkHm:!_STAN.discover.test-cluster.pings@\x05H\x03P\x01"]
+[7] 2021/01/13 14:22:50.949848 [TRC] 127.0.0.1:60748 - cid:3 - <<- [UNSUB 21 ]
+[7] 2021/01/13 14:22:50.949912 [TRC] 127.0.0.1:60748 - cid:3 - <-> [DELSUB 21]

packages/stan/data_stream/log/_dev/test/pipeline/test-log-sample.log-config.json

@@ -0,0 +1,11 @@
+{
+    "fields": {
+        "@timestamp": "2020-04-28T11:07:58.223Z",
+        "ecs": {
+            "version": "1.5.0"
+        }
+    },
+    "dynamic_fields": {
+        "event.ingested": ".*"
+    }
+}