Added working example

.gitignore

@@ -0,0 +1 @@
+.idea

Dockerfile

@@ -0,0 +1,15 @@
+FROM docker.elastic.co/logstash/logstash:6.2.2
+
+MAINTAINER "Kristofer Borgström"
+
+LABEL name="logstash multi-pipeline indexer example with xpack"
+LABEL vendor="Elastic"
+
+RUN chmod a+rwx /usr/share/logstash/data/
+
+RUN rm -f /usr/share/logstash/pipeline/logstash.conf
+
+# Add new pipelines to this yaml file and a matching folder defining the pipeline in conf.d
+COPY config/pipelines.yml /usr/share/logstash/pipeline/pipelines.yml
+COPY config/logstash.yml /usr/share/logstash/config/logstash.yml
+COPY conf.d/ /usr/share/logstash/pipeline/

README.md

@@ -0,0 +1,9 @@
+# Example use of official xpack Logstash docker image using multiple pipelines
+
+Tested using docker-compose and Openshift 3.6 & 3.7 (Kubernetes 1.6/1.7)
+
+## Pipelines
+This image supports multiple pipelines. Each pipeline needs:
+* An entry in config/pipelines.yml 
+* A pipeline defined in conf.d/<pipeline-name>
+  * As a minimum it needs 00-input.conf and 70-output.conf, please see existing pipelines like docker for examples

conf.d/pipe1/00-input.conf

@@ -0,0 +1,5 @@
+input {
+  syslog {
+    id => "my_plugin_id"
+  }
+}

conf.d/pipe1/10-filter.conf

@@ -0,0 +1 @@
+## Preprocessing filter

conf.d/pipe1/20-app.conf

@@ -0,0 +1 @@
+#Add grok filters etc. here

conf.d/pipe1/70-out.conf

@@ -0,0 +1,11 @@
+output {
+
+  elasticsearch {
+    id => "pipe1"
+    hosts => "${ELASTICSEARCH}"
+    user => "logstash" # Should not be the same as for monitoring, needs access to appropriate indexes.
+    password => "${LOGSTASH_PASSWD}"
+    index => "logstash-pipe1-%{+YYYY.MM.dd}"
+  }
+}
+

conf.d/pipe2/00-input.conf

@@ -0,0 +1,5 @@
+input {
+  syslog {
+    id => "my_plugin_id"
+  }
+}

conf.d/pipe2/10-filter.conf

@@ -0,0 +1 @@
+## Preprocessing filter

conf.d/pipe2/20-app.conf

@@ -0,0 +1 @@
+#Add grok filters etc. here

conf.d/pipe2/70-out.conf

@@ -0,0 +1,11 @@
+output {
+
+  elasticsearch {
+    id => "pipe2"
+    hosts => "${ELASTICSEARCH}"
+    user => "logstash" # Should not be the same as for monitoring, needs access to appropriate indexes.
+    password => "${LOGSTASH_PASSWD}"
+    index => "logstash-pipe2-%{+YYYY.MM.dd}"
+  }
+}
+

config/logstash.yml

@@ -0,0 +1,21 @@
+# ------------ Data path ------------------
+#
+# Which directory should be used by logstash and its plugins
+# for any persistent needs. Defaults to LOGSTASH_HOME/data
+#
+path.data: /usr/share/logstash/data/
+
+# ------------ Pipeline Configuration Settings --------------
+#
+# Where to fetch the pipeline configuration for the main pipeline
+#
+path.settings: /usr/share/logstash/pipeline/
+
+# ------------ Dead-Letter Queue Settings --------------
+# Flag to turn on dead-letter queue.
+#
+dead_letter_queue.enable: false
+
+xpack.monitoring.elasticsearch.url: http://${ELASTICSEARCH}:9200
+xpack.monitoring.elasticsearch.username: logstash_system # Make sure to use this default user for monitoring
+xpack.monitoring.elasticsearch.password: changeme

config/pipelines.yml

@@ -0,0 +1,4 @@
+- pipeline.id: pipeline1
+  path.config: "/usr/share/logstash/pipeline/pipe1/*.conf"
+- pipeline.id: pipeline2
+  path.config: "/usr/share/logstash/pipeline/pipe2/*.conf"

docker-compose.yaml

@@ -0,0 +1,8 @@
+version: '2'
+services:
+  logstash:
+    image: kribor/docker-logstash-indexer-example:6.2.2
+    build: .
+    environment:
+      - LOGSTASH_PASSWD=changeme
+      - ELASTICSEARCH=127.0.0.1