logstash between ampbeat and elasticsearch

cluster/agent/stacks/02-logstash.amp.yml

@@ -0,0 +1,30 @@
+version: "3"
+
+networks:
+  default:
+    external:
+      name: ampnet
+
+volumes:
+  logstash-data:
+
+services:
+
+  logstash:
+    image: appcelerator/amp-logstash:5.5.0
+    networks:
+      - default
+    volumes:
+      - logstash-data:/usr/share/logstash/data
+    deploy:
+      mode: replicated
+      replicas: 1
+      labels:
+        io.amp.role: "infrastructure"
+      placement:
+        constraints:
+        - node.labels.amp.type.core == true
+    labels:
+      io.amp.role: "infrastructure"
+      amp.service.stabilize.delay: "5s"
+      amp.service.stabilize.timeout: "20s"

cluster/agent/stacks/02-logstash.test.yml

@@ -0,0 +1,28 @@
+version: "3.1"
+
+networks:
+  default:
+    external:
+      name: ampnet
+
+services:
+
+  logstash:
+    image: appcelerator/alpine:3.6.0
+    networks:
+      - default
+    command: ["curl", "-sfm", "5", "${AMP_STACK:-amp}_logstash:9600/"]
+    labels:
+      io.amp.role: "infrastructure"
+      io.amp.test:
+    deploy:
+      mode: replicated
+      replicas: 1
+      labels:
+        io.amp.role: "infrastructure"
+        io.amp.test:
+      placement:
+        constraints:
+        - node.labels.amp.type.core == true
+      restart_policy:
+        condition: none

cmd/ampbeat/ampbeat.yml

@@ -1,6 +1,6 @@
-################### Ampbeat Configuration Example #########################
+########################## Ampbeat Configuration ##############################
 
-############################# Ampbeat ######################################
+################################ Ampbeat ######################################
 
 ampbeat:
   # Defines how often an event is sent to the output
@@ -27,19 +27,23 @@ ampbeat:
 # Multiple outputs may be used.
 
 #-------------------------- Elasticsearch output ------------------------------
-output.elasticsearch:
-  # Array of hosts to connect to.
-  hosts: ["elasticsearch:9200"]
+#output.elasticsearch:
+  ## Array of hosts to connect to.
+  #hosts: ["elasticsearch:9200"]
+  #template.versions.2x.enabled: false
 
-  # Optional protocol and basic auth credentials.
-  #protocol: "https"
-  #username: "elastic"
-  #password: "changeme"
+  ## Optional protocol and basic auth credentials.
+  ##protocol: "https"
+  ##username: "elastic"
+  ##password: "changeme"
 
 #----------------------------- Logstash output --------------------------------
-#output.logstash:
+output.logstash:
   # The Logstash hosts
-  #hosts: ["localhost:5044"]
+  hosts: ["logstash:5044"]
+  index: "ampbeat"
+  pipelining: 8
+  bulk_max_size: 2048
 
   # Optional SSL. By default is off.
   # List of root certificates for HTTPS server verifications
@@ -51,6 +55,13 @@ output.elasticsearch:
   # Client Certificate Key
   #ssl.key: "/etc/pki/client/cert.key"
 
+#------------------------------- Redis output ---------------------------------
+#output.redis:
+  #hosts: ["redis"]
+  #key: "ampbeat"
+  #datatype: "list"
+  #bulk_max_size: 2048
+
 #================================ Logging =====================================
 
 # Sets log level. The default log level is info.

images/logstash/Dockerfile

@@ -0,0 +1,34 @@
+FROM golang:1.8-alpine as buildenv2yaml
+
+RUN apk add --no-cache build-base ca-certificates git
+RUN go get gopkg.in/yaml.v2
+COPY env2yaml /go/src/env2yaml
+RUN cd /go/src && go build -o /bin/env2yaml env2yaml
+
+FROM appcelerator/alpine:3.6.0 as buildlogstash
+ENV ELASTIC_VERSION 5.5.0
+RUN curl -L https://artifacts.elastic.co/downloads/logstash/logstash-$ELASTIC_VERSION.tar.gz -o /tmp/logstash-$ELASTIC_VERSION.tar.gz && \
+    tar xzf /tmp/logstash-$ELASTIC_VERSION.tar.gz -C /usr/share && \
+    mv /usr/share/logstash-$ELASTIC_VERSION /usr/share/logstash && \
+    rm -f /usr/share/logstash/bin/logstash*exe /usr/share/logstash/bin/logstash*bat && \
+    rm /tmp/logstash-$ELASTIC_VERSION.tar.gz
+ADD pipeline/ /usr/share/logstash/pipeline/
+ADD config/ /usr/share/logstash/config/
+ADD template/ /usr/share/logstash/template/
+RUN adduser -D -h /usr/share/logstash -s /sbin/nologin elastico
+RUN chown -R elastico:elastico /usr/share/logstash
+
+FROM appcelerator/alpine:3.6.0
+RUN apk update && apk upgrade && apk add openjdk8-jre bind-tools && rm -rf /var/cache/apk/*
+ENV PATH /bin:/usr/share/logstash/bin:$PATH
+ENV ELASTIC_VERSION 5.5.0
+RUN adduser -D -h /usr/share/logstash -s /sbin/nologin elastico
+RUN chown -R elastico:elastico /usr/share/logstash
+COPY --from=buildenv2yaml /bin/env2yaml /usr/local/bin/env2yaml
+COPY --from=buildlogstash /usr/share/logstash /usr/share/logstash
+WORKDIR /usr/share/logstash
+COPY /bin/docker-entrypoint.sh /usr/local/bin/
+VOLUME /usr/share/logstash/data
+EXPOSE 5044 9600
+ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
+CMD ["-f", "/usr/share/logstash/pipeline"]

images/logstash/bin/docker-entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+# Map environment variables to entries in logstash.yml.
+# Note that this will mutate logstash.yml in place if any such settings are found.
+# This may be undesirable, especially if logstash.yml is bind-mounted from the
+# host system.
+env2yaml /usr/share/logstash/config/logstash.yml
+
+if [[ -z $1 ]] || [[ ${1:0:1} == '-' ]] ; then
+  exec logstash "$@"
+else
+  exec "$@"
+fi

images/logstash/config/jvm.options

@@ -0,0 +1,79 @@
+## JVM configuration
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+#-Xms{{ .XMS | default "256" }}m
+#-Xmx{{ .XMX | default "1024" }}m
+-Xms256m
+-Xmx1024m
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseParNewGC
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# disable calls to System#gc
+-XX:+DisableExplicitGC
+
+## Locale
+# Set the locale language
+#-Duser.language=en
+
+# Set the locale country
+#-Duser.country=US
+
+# Set the locale variant, if any
+#-Duser.variant=
+
+## basic
+
+# set the I/O temp directory
+#-Djava.io.tmpdir=$HOME
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+#-Djna.nosys=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof
+
+## GC logging
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${LS_GC_LOG_FILE}
+
+# Entropy source for randomness
+-Djava.security.egd=file:/dev/urandom

images/logstash/config/log4j2.properties

@@ -0,0 +1,19 @@
+status = error
+name = LogstashPropertiesConfig
+
+appender.console.type = Console
+appender.console.name = plain_console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %m%n
+
+appender.json_console.type = Console
+appender.json_console.name = json_console
+appender.json_console.layout.type = JSONLayout
+appender.json_console.layout.compact = true
+appender.json_console.layout.eventEol = true
+
+rootLogger.level = ${sys:ls.log.level}
+rootLogger.appenderRef.console.ref = ${sys:ls.log.format}_console
+
+slowlog.threshold.warn: 2s
+slowlog.threshold.info: 1s

images/logstash/config/logstash.yml

@@ -0,0 +1,30 @@
+# node.name: hostname
+# path.data: LOGSTASH_HOME/data
+# pipeline.workers: 2
+# pipeline.output.workers: 1
+# pipeline.batch.size: 125
+# pipeline.batch.delay: 5
+# pipeline.unsafe_shutdown: false
+path.config: /usr/share/logstash/pipeline
+# config.string:
+# config.test_and_exit: false
+config.reload.automatic: true
+config.reload.interval: 3
+# config.debug: false
+# config.support_escapes: false
+queue.type: persisted
+# path.queue:
+# queue.page_capacity: 250mb
+# queue.max_events: 0
+# queue.max_bytes: 1024mb
+# queue.checkpoint.acks: 1024
+# queue.checkpoint.writes: 1024
+# queue.checkpoint.interval: 1000
+# dead_letter_queue.enable: false
+# path.dead_letter_queue:
+http.host: "0.0.0.0"
+# http.port: 9600-9700
+# log.level: info
+# path.logs:
+# path.plugins: []
+# xpack.monitoring.enabled: false