diff --git a/vamp/Dockerfile b/vamp/Dockerfile index 5731ec4..d248a6c 100644 --- a/vamp/Dockerfile +++ b/vamp/Dockerfile @@ -1,16 +1,75 @@ -FROM java:openjdk-8-jre-alpine +FROM magneticio/java:openjdk-8-jre-alpine + +# https://github.com/peterbourgon/runsvinit +ENV RUNSVINIT_URL=https://github.com/peterbourgon/runsvinit/releases/download/v2.0.0/runsvinit-linux-amd64.tgz + +ENV METRICBEAT_VER=5.2.1 +ENV METRICBEAT_URL=https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-${METRICBEAT_VER}-linux-x86_64.tar.gz + +ENV HTTPBEAT_VER=4.0.0 +ENV HTTPBEAT_URL=https://github.com/christiangalsterer/httpbeat/releases/download/${HTTPBEAT_VER}/httpbeat-${HTTPBEAT_VER}-linux-x86_64.tar.gz + +RUN ALPINE_GLIBC_BASE_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download" && \ + ALPINE_GLIBC_PACKAGE_VERSION="2.23-r3" && \ + ALPINE_GLIBC_BASE_PACKAGE_FILENAME="glibc-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ + ALPINE_GLIBC_BIN_PACKAGE_FILENAME="glibc-bin-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ + ALPINE_GLIBC_I18N_PACKAGE_FILENAME="glibc-i18n-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \ + apk add --no-cache --virtual=.build-dependencies wget ca-certificates && \ + wget \ + "https://raw.githubusercontent.com/andyshinn/alpine-pkg-glibc/master/sgerrand.rsa.pub" \ + -O "/etc/apk/keys/sgerrand.rsa.pub" && \ + wget \ + "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ + "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ + "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \ + apk add --no-cache \ + "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ + "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ + "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \ + \ + rm "/etc/apk/keys/sgerrand.rsa.pub" && \ + /usr/glibc-compat/bin/localedef --force --inputfile POSIX --charmap UTF-8 C.UTF-8 || true && \ + echo "export LANG=C.UTF-8" > /etc/profile.d/locale.sh && \ + \ + apk del glibc-i18n && \ + \ + rm "/root/.wget-hsts" && \ + apk del .build-dependencies && \ + rm \ + "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \ + "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \ + "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" + +ENV LANG=C.UTF-8 + +RUN set -xe \ + && apk add --no-cache \ + bash \ + curl \ + runit \ + && curl --location --silent --show-error $RUNSVINIT_URL --output - | tar zxf - -C /sbin \ + && chown 0:0 /sbin/runsvinit \ + && chmod 0775 /sbin/runsvinit \ + \ + && mkdir -p /usr/local/metricbeat/ \ + && curl --location --silent --show-error $METRICBEAT_URL --output - | tar zxf - -C /tmp \ + && mv /tmp/metricbeat-${METRICBEAT_VER}-linux-x86_64/metricbeat /usr/local/metricbeat/ \ + && rm -rf /tmp/metricbeat-${METRICBEAT_VER}-linux-x86_64 \ + \ + && mkdir -p /usr/local/httpbeat/ \ + && curl --location --silent --show-error $HTTPBEAT_URL --output - | tar zxf - -C /tmp \ + && mv /tmp/httpbeat-5.2.1-SNAPSHOT-linux-x86_64/httpbeat /usr/local/httpbeat/ \ + && rm -rf /tmp/httpbeat-5.2.1-SNAPSHOT-linux-x86_64 RUN mkdir -p /usr/local/vamp/conf ADD vamp-artifacts /usr/local/vamp/artifacts ADD vamp-ui /usr/local/vamp/ui ADD vamp /usr/local/vamp/bin -ADD logback.xml /usr/local/vamp/ -ADD vamp.sh /usr/local/vamp/vamp -RUN chmod +x /usr/local/vamp/vamp +ADD files/ / ENV VAMP_REST_API_UI_DIRECTORY /usr/local/vamp/ui ENV VAMP_REST_API_UI_INDEX /usr/local/vamp/ui/index.html VOLUME /usr/local/vamp/conf -CMD ["/usr/local/vamp/vamp"] +CMD ["/sbin/runsvinit"] diff --git a/vamp/README.md b/vamp/README.md index 00a4b6f..fcfeffd 100644 --- a/vamp/README.md +++ b/vamp/README.md @@ -5,11 +5,17 @@ This can be done in few way: - providing `application.conf` file and/or - setting environment variables + Let's say `application.conf` is stored in `/MY/APP/CONF/PATH` directory (absolute path), then: -``` +```shell docker run -v /MY/APP/CONF/PATH:/usr/local/vamp/conf magneticio/vamp:katana ``` +For metric collection to work Elasticsearch URL need to be provided through the environment variable `VAMP_ELASTICSEARCH_URL`. +```shell +docker run -e VAMP_ELASTICSEARCH_URL=http://elasticsearch.example.com:9200 -v /MY/APP/CONF/PATH:/usr/local/vamp/conf magneticio/vamp:katana +``` + Required minimal configuration is: ```yaml vamp { @@ -74,4 +80,4 @@ An example can be found also in `Dockerfile`: ```bash ENV VAMP_REST_API_UI_DIRECTORY /usr/local/vamp/ui ENV VAMP_REST_API_UI_INDEX /usr/local/vamp/ui/index.html -``` \ No newline at end of file +``` diff --git a/vamp/alpine/Dockerfile b/vamp/alpine/Dockerfile new file mode 100644 index 0000000..ef3d07c --- /dev/null +++ b/vamp/alpine/Dockerfile @@ -0,0 +1,32 @@ +# Alpine 3.5 version of java:openjdk-8-jre-alpine +# From: https://github.com/docker-library/openjdk/blob/9a0822673dffd3e5ba66f18a8547aa60faed6d08/8-jre/alpine/Dockerfile + +FROM alpine:3.5 + +# A few problems with compiling Java from source: +# 1. Oracle. Licensing prevents us from redistributing the official JDK. +# 2. Compiling OpenJDK also requires the JDK to be installed, and it gets +# really hairy. + +# Default to UTF-8 file.encoding +ENV LANG C.UTF-8 + +# add a simple script that can auto-detect the appropriate JAVA_HOME value +# based on whether the JDK or only the JRE is installed +RUN { \ + echo '#!/bin/sh'; \ + echo 'set -e'; \ + echo; \ + echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \ + } > /usr/local/bin/docker-java-home \ + && chmod +x /usr/local/bin/docker-java-home +ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk/jre +ENV PATH $PATH:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin + +ENV JAVA_VERSION 8u111 +ENV JAVA_ALPINE_VERSION 8.121.13-r0 + +RUN set -x \ + && apk add --no-cache \ + openjdk8-jre="$JAVA_ALPINE_VERSION" \ + && [ "$JAVA_HOME" = "$(docker-java-home)" ] diff --git a/vamp/alpine/Makefile b/vamp/alpine/Makefile new file mode 100644 index 0000000..d91a6f2 --- /dev/null +++ b/vamp/alpine/Makefile @@ -0,0 +1,7 @@ +all: build + +build: + docker build --tag magneticio/java:openjdk-8-jre-alpine . + +clean: + docker rmi magneticio/java:openjdk-8-jre-alpine diff --git a/vamp/files/etc/service/httpbeat/run b/vamp/files/etc/service/httpbeat/run new file mode 100755 index 0000000..926b96a --- /dev/null +++ b/vamp/files/etc/service/httpbeat/run @@ -0,0 +1,13 @@ +#! /bin/bash +handle() { echo "httpbeat/run: got signal"; exit; } +trap handle SIGINT + +if [[ -z $VAMP_ELASTICSEARCH_URL ]] ; then + sv stop httpbeat + exit +fi + +[[ -e /usr/local/httpbeat/httpbeat.yml ]] || exit 1 + +exec 2>&1 +exec /usr/local/httpbeat/httpbeat -c /usr/local/httpbeat/httpbeat.yml diff --git a/vamp/files/etc/service/metricbeat/run b/vamp/files/etc/service/metricbeat/run new file mode 100755 index 0000000..2ae1a6b --- /dev/null +++ b/vamp/files/etc/service/metricbeat/run @@ -0,0 +1,13 @@ +#! /bin/bash +handle() { echo "metricbeat/run: got signal"; exit; } +trap handle SIGINT + +if [[ -z $VAMP_ELASTICSEARCH_URL ]] ; then + sv stop metricbeat + exit +fi + +[[ -e /usr/local/metricbeat/metricbeat.yml ]] || exit 1 + +exec 2>&1 +exec /usr/local/metricbeat/metricbeat -c /usr/local/metricbeat/metricbeat.yml diff --git a/vamp/files/etc/service/vamp/run b/vamp/files/etc/service/vamp/run new file mode 100755 index 0000000..1673bc5 --- /dev/null +++ b/vamp/files/etc/service/vamp/run @@ -0,0 +1,6 @@ +#! /bin/bash +handle() { echo "vamp/run: got signal"; exit; } +trap handle SIGINT + +exec 2>&1 +exec /usr/local/vamp/vamp diff --git a/vamp/files/usr/local/httpbeat/httpbeat.template-es2x.json b/vamp/files/usr/local/httpbeat/httpbeat.template-es2x.json new file mode 100644 index 0000000..0e1a589 --- /dev/null +++ b/vamp/files/usr/local/httpbeat/httpbeat.template-es2x.json @@ -0,0 +1,143 @@ +{ + "mappings": { + "_default_": { + "_all": { + "norms": { + "enabled": false + } + }, + "_meta": { + "version": "5.2.1" + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "version": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "instance_id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "machine_type": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "project_id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "provider": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "region": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + } + }, + "request": { + "properties": { + "body": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "headers": { + "properties": {}, + "type": "nested" + }, + "method": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "url": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "response": { + "properties": { + "body": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "code": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "headers": { + "properties": {}, + "type": "nested" + }, + "jsonBody": { + "properties": {}, + "type": "nested" + } + } + }, + "tags": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + }, + "order": 0, + "settings": { + "index.refresh_interval": "5s" + }, + "template": "httpbeat-*" +} \ No newline at end of file diff --git a/vamp/files/usr/local/httpbeat/httpbeat.template.json b/vamp/files/usr/local/httpbeat/httpbeat.template.json new file mode 100644 index 0000000..dcaf7da --- /dev/null +++ b/vamp/files/usr/local/httpbeat/httpbeat.template.json @@ -0,0 +1,126 @@ +{ + "mappings": { + "_default_": { + "_all": { + "norms": false + }, + "_meta": { + "version": "5.2.1" + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "request": { + "properties": { + "body": { + "ignore_above": 1024, + "type": "keyword" + }, + "headers": { + "properties": {}, + "type": "nested" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "headers": { + "properties": {}, + "type": "nested" + }, + "jsonBody": { + "properties": {}, + "type": "nested" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "order": 0, + "settings": { + "index.mapping.total_fields.limit": 10000, + "index.refresh_interval": "5s" + }, + "template": "httpbeat-*" +} \ No newline at end of file diff --git a/vamp/files/usr/local/httpbeat/httpbeat.yml b/vamp/files/usr/local/httpbeat/httpbeat.yml new file mode 100644 index 0000000..241fc77 --- /dev/null +++ b/vamp/files/usr/local/httpbeat/httpbeat.yml @@ -0,0 +1,23 @@ +httpbeat: + + hosts: + - + # Optional cron expression, defines when to poll the host endpoint. + # Default is every 1 minute. + schedule: "@every 5s" + # Vamp API info endpoint + url: http://127.0.0.1:8080/api/v1/info?on=jvm + method: get + headers: + Accept: application/json + output_format: json + +output.elasticsearch: + hosts: ["${VAMP_ELASTICSEARCH_URL}"] + index: "vamp-%{+yyyy-MM-dd}" + template.path: /usr/local/metricbeat/metricbeat.template.json + +path.home: /usr/local/httpbeat +path.config: ${path.home} +path.data: ${path.home}/data +path.logs: /var/log diff --git a/vamp/files/usr/local/metricbeat/metricbeat.template-es2x.json b/vamp/files/usr/local/metricbeat/metricbeat.template-es2x.json new file mode 100644 index 0000000..6872f7f --- /dev/null +++ b/vamp/files/usr/local/metricbeat/metricbeat.template-es2x.json @@ -0,0 +1,999 @@ +{ + "mappings": { + "_default_": { + "_all": { + "norms": { + "enabled": false + } + }, + "_meta": { + "version": "5.1.2" + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "version": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "instance_id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "machine_type": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "project_id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "provider": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "region": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + } + }, + "metricset": { + "properties": { + "host": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "module": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "rtt": { + "type": "long" + } + } + }, + "system": { + "properties": { + "core": { + "properties": { + "id": { + "type": "long" + }, + "idle": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "idle": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "diskio": { + "properties": { + "io": { + "properties": { + "time": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "read": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "time": { + "type": "long" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "write": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "time": { + "type": "long" + } + } + } + } + }, + "filesystem": { + "properties": { + "available": { + "type": "long" + }, + "device_name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "files": { + "type": "long" + }, + "free": { + "type": "long" + }, + "free_files": { + "type": "long" + }, + "mount_point": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "float" + } + } + } + } + }, + "fsstat": { + "properties": { + "count": { + "type": "long" + }, + "total_files": { + "type": "long" + }, + "total_size": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "type": "long" + } + } + } + } + }, + "load": { + "properties": { + "1": { + "type": "float" + }, + "15": { + "type": "float" + }, + "5": { + "type": "float" + }, + "norm": { + "properties": { + "1": { + "type": "float" + }, + "15": { + "type": "float" + }, + "5": { + "type": "float" + } + } + } + } + }, + "memory": { + "properties": { + "actual": { + "properties": { + "free": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "float" + } + } + } + } + }, + "free": { + "type": "long" + }, + "swap": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "float" + } + } + } + } + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "float" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, + "process": { + "properties": { + "cgroup": { + "properties": { + "blkio": { + "properties": { + "id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "path": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "total": { + "properties": { + "bytes": { + "type": "long" + }, + "ios": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "cfs": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "us": { + "type": "long" + } + } + }, + "shares": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "path": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "rt": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "runtime": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "stats": { + "properties": { + "periods": { + "type": "long" + }, + "throttled": { + "properties": { + "ns": { + "type": "long" + }, + "periods": { + "type": "long" + } + } + } + } + } + } + }, + "cpuacct": { + "properties": { + "id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "path": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "stats": { + "properties": { + "system": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "user": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "memory": { + "properties": { + "id": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "kmem": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "kmem_tcp": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "mem": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "memsw": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "path": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "stats": { + "properties": { + "active_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "active_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "cache": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memory_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memsw_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "major_page_faults": { + "type": "long" + }, + "mapped_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + }, + "pages_in": { + "type": "long" + }, + "pages_out": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "rss_huge": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "swap": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "unevictable": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "path": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "cmdline": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "cpu": { + "properties": { + "start_time": { + "type": "date" + }, + "system": { + "type": "long" + }, + "total": { + "properties": { + "pct": { + "type": "float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "type": "long" + } + } + }, + "fd": { + "properties": { + "limit": { + "properties": { + "hard": { + "type": "long" + }, + "soft": { + "type": "long" + } + } + }, + "open": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "rss": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "float" + } + } + }, + "share": { + "type": "long" + }, + "size": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "username": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "type": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + }, + "order": 0, + "settings": { + "index.refresh_interval": "5s" + }, + "template": "vamp-vga-*" +} diff --git a/vamp/files/usr/local/metricbeat/metricbeat.template.json b/vamp/files/usr/local/metricbeat/metricbeat.template.json new file mode 100644 index 0000000..ac75966 --- /dev/null +++ b/vamp/files/usr/local/metricbeat/metricbeat.template.json @@ -0,0 +1,991 @@ +{ + "mappings": { + "_default_": { + "_all": { + "norms": false + }, + "_meta": { + "version": "5.1.2" + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "meta": { + "properties": { + "cloud": { + "properties": { + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "machine_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "metricset": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "rtt": { + "type": "long" + } + } + }, + "system": { + "properties": { + "core": { + "properties": { + "id": { + "type": "long" + }, + "idle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "idle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "diskio": { + "properties": { + "io": { + "properties": { + "time": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "read": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "time": { + "type": "long" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "write": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "time": { + "type": "long" + } + } + } + } + }, + "filesystem": { + "properties": { + "available": { + "type": "long" + }, + "device_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "files": { + "type": "long" + }, + "free": { + "type": "long" + }, + "free_files": { + "type": "long" + }, + "mount_point": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "fsstat": { + "properties": { + "count": { + "type": "long" + }, + "total_files": { + "type": "long" + }, + "total_size": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "type": "long" + } + } + } + } + }, + "load": { + "properties": { + "1": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "15": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "5": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "norm": { + "properties": { + "1": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "15": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "5": { + "scaling_factor": 100, + "type": "scaled_float" + } + } + } + } + }, + "memory": { + "properties": { + "actual": { + "properties": { + "free": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "free": { + "type": "long" + }, + "swap": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, + "process": { + "properties": { + "cgroup": { + "properties": { + "blkio": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "properties": { + "bytes": { + "type": "long" + }, + "ios": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "cfs": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "us": { + "type": "long" + } + } + }, + "shares": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "rt": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "runtime": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "stats": { + "properties": { + "periods": { + "type": "long" + }, + "throttled": { + "properties": { + "ns": { + "type": "long" + }, + "periods": { + "type": "long" + } + } + } + } + } + } + }, + "cpuacct": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "properties": { + "system": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "user": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kmem": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "kmem_tcp": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "mem": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "memsw": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "properties": { + "active_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "active_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "cache": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memory_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memsw_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "major_page_faults": { + "type": "long" + }, + "mapped_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + }, + "pages_in": { + "type": "long" + }, + "pages_out": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "rss_huge": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "swap": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "unevictable": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cmdline": { + "ignore_above": 1024, + "type": "keyword" + }, + "cpu": { + "properties": { + "start_time": { + "type": "date" + }, + "system": { + "type": "long" + }, + "total": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "type": "long" + } + } + }, + "fd": { + "properties": { + "limit": { + "properties": { + "hard": { + "type": "long" + }, + "soft": { + "type": "long" + } + } + }, + "open": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "rss": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "share": { + "type": "long" + }, + "size": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "order": 0, + "settings": { + "index.refresh_interval": "5s" + }, + "template": "vamp-*" +} diff --git a/vamp/files/usr/local/metricbeat/metricbeat.yml b/vamp/files/usr/local/metricbeat/metricbeat.yml new file mode 100644 index 0000000..973899a --- /dev/null +++ b/vamp/files/usr/local/metricbeat/metricbeat.yml @@ -0,0 +1,24 @@ +metricbeat.modules: +- module: system + metricsets: + - cpu # CPU stats + - load # System Load stats + - filesystem # Per filesystem stats + - fsstat # File system summary stats + - memory # Memory stats + - network # Network stats + - process # Per process stats + enabled: true + period: 1s + processes: ['.*'] + tags: ["vamp"] + +output.elasticsearch: + hosts: ["${VAMP_ELASTICSEARCH_URL}"] + index: "vamp-%{+yyyy-MM-dd}" + template.path: /usr/local/metricbeat/metricbeat.template.json + +path.home: /usr/local/metricbeat +path.config: ${path.home} +path.data: ${path.home}/data +path.logs: /var/log diff --git a/vamp/logback.xml b/vamp/files/usr/local/vamp/logback.xml similarity index 100% rename from vamp/logback.xml rename to vamp/files/usr/local/vamp/logback.xml diff --git a/vamp/vamp.sh b/vamp/files/usr/local/vamp/vamp similarity index 100% rename from vamp/vamp.sh rename to vamp/files/usr/local/vamp/vamp diff --git a/vamp/make.sh b/vamp/make.sh index 00d2c84..ce49543 100755 --- a/vamp/make.sh +++ b/vamp/make.sh @@ -54,4 +54,5 @@ pull vamp-artifacts join vamp-kubernetes join vamp-elasticsearch -cp -f ${dir}/Dockerfile ${dir}/vamp.sh ${dir}/logback.xml ${target}/ +cp -Rf ${dir}/files ${target}/ +cp -f ${dir}/Dockerfile ${target}/