various fixes

.kitchen.yml

@@ -13,10 +13,6 @@ platforms:
 - name: ubuntu-14
 - name: ubuntu-15
 
-transport:
-  name: ssh
-  compression: none
-
 provisioner:
   name: chef_zero
   require_chef_omnibus: '11'

CHANGELOG.md

@@ -1,35 +1,30 @@
-ellk CHANGELOG
-=============
-
-This file is used to list changes made in each version of the ellk cookbook.
-
-
-0.3.2
------
-- Jacob Dearing - Cosmtic updates and Berkshelf pinning to avoid bug in Runit
-			    - https://github.com/dearing/ellk/issues/22
-
-0.3.0
------
-- Jacob Dearing - Release-Ready for feedback. :8ball: 
-
-0.2.5
------
-- Jacob Dearing - Kibana looking more Release-Ready :+1:
-
-0.2.4
------
-- Jacob Dearing - Logstash kinda Release-Ready :+1:
-
-0.2.3
------
-- Jacob Dearing - Logstash-Forwarder Release-Ready :+1:
-
-0.2.0
------
-- Jacob Dearing - Pre-Release
-
-- - -
-Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown.
-
-The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown.
+# Releases
+- [releases]
+- [supermarket]
+
+# Change Log
+
+## [Unreleased][unreleased]
+### Changed
+- better config defaults for logstash & logstash-forwader
+- logstash-forwarder resource no longer requires a key
+- logstash resource now requires both a key and cert
+
+## [v0.3.2] 
+### Changed
+- Cosmtic updates and Berkshelf pinning to avoid bug in Runit; see #22
+
+## [v0.3.1] 
+### Changed
+- Submission for feedback
+
+## [v0.3.0] 
+### Changed
+- Submission for feedback
+
+[unreleased]: https://github.com/dearing/ellk/compare/v3.2.0...HEAD
+[v0.3.2]: https://github.com/dearing/ellk/compare/v0.3.2...v0.3.3
+[v0.3.1]: https://github.com/dearing/ellk/compare/v0.3.1...v0.3.2
+[v0.3.0]: https://github.com/dearing/ellk/compare/v0.3.0...v0.3.1
+[releases]: https://github.com/dearing/ellk/releases
+[supermarket]: https://supermarket.chef.io/cookbooks/ellk

metadata.rb

@@ -4,11 +4,11 @@
 license 'MIT'
 description 'Library to handle Elasticsearch, Logstash, Logstash-Forwarder & Kibana'
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version '0.3.2'
+version '0.3.3'
 
 %w(ubuntu debian centos redhat amazon scientific oracle enterpriseenterprise).each do |os|
   supports os
 end
 
 depends 'ark'
-depends 'runit', '= 1.5.18' # until https://github.com/hw-cookbooks/runit/issues/142
+depends 'runit', '= 1.5.18'

templates/default/logstash/logstash.conf.erb

@@ -11,12 +11,51 @@ input {
 }
 
 filter {
-## ADD FILTERS HERE 
+ if [type] == "syslog" {
+    grok {
+      overwrite => "message"
+      match => [
+        "message",
+        "%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}"
+      ]
+    }
+    syslog_pri { }
+    date {
+      # season to taste for your own syslog format(s)
+      match => [
+        "timestamp",
+        "MMM  d HH:mm:ss",
+        "MMM dd HH:mm:ss",
+        "ISO8601"
+      ]
+    }
+  }
+
+  if [type] == "apache-access" {
+      grok {
+        patterns_dir => ["./config/patterns"]
+        match => ["message", "%{COMBINEDAPACHELOG}" ]
+      }
+      date {
+        match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss" ]
+      }
+  }
+
+  if [type] == "nginx-access" {
+      grok {
+        patterns_dir => ["./config/patterns"]
+        match => ["message", "%{NGINXACCESS}" ]
+      }
+      date {
+        match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss" ]
+      }
+  }
 }
 
+
 # TODO: logstash doesn't convert 1:1 with ruby or json, hash passing will have to wait
 # for now we consider all elasticsearch nodes have a logstash service on the same machine
 output {
   elasticsearch { host => localhost}
-  stdout { codec => rubydebug }
+  stdout { codec => json }
 }

test/cookbooks/ellktest/recipes/default.rb

@@ -22,7 +22,7 @@
   to '/usr/bin/sv'
 end
 
-# create certs (subject = localhost)
+# create certs (SN = localhost)
 secrets = Chef::DataBagItem.load('secrets', 'logstash')
 logstash_key = Base64.decode64(secrets['key'])
 file '/tmp/logstash.key' do
@@ -55,9 +55,8 @@
 ## LOGSTASH-FORWARDER
 logstash_forwarder 'default' do
   crt_location '/tmp/logstash.crt'
-  # key_location '/tmp/logstash.key'
   logstash_servers ['localhost:5043']
-  files [{ 'paths' => ['/var/log/messages', '/var/log/*log', '/var/log/kibana/current'], 'fields' => { 'type' => 'syslog' } }]
+  files [{ 'paths' => ['/var/log/messages', '/var/log/*log'], 'fields' => { 'type' => 'syslog' } }]
 end
 
 ## KIBANA

test/cookbooks/ellktest/templates/default/logstash/logstash.conf.erb

@@ -11,22 +11,31 @@ input {
 }
 
 filter {
-  if [type] == "syslog" {
+ if [type] == "syslog" {
     grok {
-      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
-      add_field => [ "received_at", "%{@timestamp}" ]
-      add_field => [ "received_from", "%{host}" ]
+      overwrite => "message"
+      match => [
+        "message",
+        "%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}"
+      ]
     }
     syslog_pri { }
     date {
-      match => [ "syslog_timestamp", "MMM  d HH:mm:ss", "MMM dd HH:mm:ss" ]
+      # season to taste for your own syslog format(s)
+      match => [
+        "timestamp",
+        "MMM  d HH:mm:ss",
+        "MMM dd HH:mm:ss",
+        "ISO8601"
+      ]
     }
   }
 }
 
+
 # TODO: logstash doesn't convert 1:1 with ruby or json, hash passing will have to wait
 # for now we consider all elasticsearch nodes have a logstash service on the same machine
 output {
   elasticsearch { host => localhost}
-  stdout { codec => rubydebug }
-}
+  stdout { codec => json }
+}