diff --git a/.kitchen.yml b/.kitchen.yml index 8a3480a..538eb12 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -13,10 +13,6 @@ platforms: - name: ubuntu-14 - name: ubuntu-15 -transport: - name: ssh - compression: none - provisioner: name: chef_zero require_chef_omnibus: '11' diff --git a/CHANGELOG.md b/CHANGELOG.md index fde286e..672d2ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,35 +1,30 @@ -ellk CHANGELOG -============= - -This file is used to list changes made in each version of the ellk cookbook. - - -0.3.2 ------ -- Jacob Dearing - Cosmtic updates and Berkshelf pinning to avoid bug in Runit - - https://github.com/dearing/ellk/issues/22 - -0.3.0 ------ -- Jacob Dearing - Release-Ready for feedback. :8ball: - -0.2.5 ------ -- Jacob Dearing - Kibana looking more Release-Ready :+1: - -0.2.4 ------ -- Jacob Dearing - Logstash kinda Release-Ready :+1: - -0.2.3 ------ -- Jacob Dearing - Logstash-Forwarder Release-Ready :+1: - -0.2.0 ------ -- Jacob Dearing - Pre-Release - -- - - -Check the [Markdown Syntax Guide](http://daringfireball.net/projects/markdown/syntax) for help with Markdown. - -The [Github Flavored Markdown page](http://github.github.com/github-flavored-markdown/) describes the differences between markdown on github and standard markdown. +# Releases +- [releases] +- [supermarket] + +# Change Log + +## [Unreleased][unreleased] +### Changed +- better config defaults for logstash & logstash-forwader +- logstash-forwarder resource no longer requires a key +- logstash resource now requires both a key and cert + +## [v0.3.2] +### Changed +- Cosmtic updates and Berkshelf pinning to avoid bug in Runit; see #22 + +## [v0.3.1] +### Changed +- Submission for feedback + +## [v0.3.0] +### Changed +- Submission for feedback + +[unreleased]: https://github.com/dearing/ellk/compare/v3.2.0...HEAD +[v0.3.2]: https://github.com/dearing/ellk/compare/v0.3.2...v0.3.3 +[v0.3.1]: https://github.com/dearing/ellk/compare/v0.3.1...v0.3.2 +[v0.3.0]: https://github.com/dearing/ellk/compare/v0.3.0...v0.3.1 +[releases]: https://github.com/dearing/ellk/releases +[supermarket]: https://supermarket.chef.io/cookbooks/ellk \ No newline at end of file diff --git a/metadata.rb b/metadata.rb index 17b7205..077df1c 100644 --- a/metadata.rb +++ b/metadata.rb @@ -4,11 +4,11 @@ license 'MIT' description 'Library to handle Elasticsearch, Logstash, Logstash-Forwarder & Kibana' long_description IO.read(File.join(File.dirname(__FILE__), 'README.md')) -version '0.3.2' +version '0.3.3' %w(ubuntu debian centos redhat amazon scientific oracle enterpriseenterprise).each do |os| supports os end depends 'ark' -depends 'runit', '= 1.5.18' # until https://github.com/hw-cookbooks/runit/issues/142 +depends 'runit', '= 1.5.18' diff --git a/templates/default/logstash/logstash.conf.erb b/templates/default/logstash/logstash.conf.erb index 07d7cb8..54f26c4 100644 --- a/templates/default/logstash/logstash.conf.erb +++ b/templates/default/logstash/logstash.conf.erb @@ -11,12 +11,51 @@ input { } filter { -## ADD FILTERS HERE + if [type] == "syslog" { + grok { + overwrite => "message" + match => [ + "message", + "%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}" + ] + } + syslog_pri { } + date { + # season to taste for your own syslog format(s) + match => [ + "timestamp", + "MMM d HH:mm:ss", + "MMM dd HH:mm:ss", + "ISO8601" + ] + } + } + + if [type] == "apache-access" { + grok { + patterns_dir => ["./config/patterns"] + match => ["message", "%{COMBINEDAPACHELOG}" ] + } + date { + match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss" ] + } + } + + if [type] == "nginx-access" { + grok { + patterns_dir => ["./config/patterns"] + match => ["message", "%{NGINXACCESS}" ] + } + date { + match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z", "dd/MMM/yyyy:HH:mm:ss" ] + } + } } + # TODO: logstash doesn't convert 1:1 with ruby or json, hash passing will have to wait # for now we consider all elasticsearch nodes have a logstash service on the same machine output { elasticsearch { host => localhost} - stdout { codec => rubydebug } + stdout { codec => json } } diff --git a/test/cookbooks/ellktest/recipes/default.rb b/test/cookbooks/ellktest/recipes/default.rb index b9a2f73..812e327 100644 --- a/test/cookbooks/ellktest/recipes/default.rb +++ b/test/cookbooks/ellktest/recipes/default.rb @@ -22,7 +22,7 @@ to '/usr/bin/sv' end -# create certs (subject = localhost) +# create certs (SN = localhost) secrets = Chef::DataBagItem.load('secrets', 'logstash') logstash_key = Base64.decode64(secrets['key']) file '/tmp/logstash.key' do @@ -55,9 +55,8 @@ ## LOGSTASH-FORWARDER logstash_forwarder 'default' do crt_location '/tmp/logstash.crt' - # key_location '/tmp/logstash.key' logstash_servers ['localhost:5043'] - files [{ 'paths' => ['/var/log/messages', '/var/log/*log', '/var/log/kibana/current'], 'fields' => { 'type' => 'syslog' } }] + files [{ 'paths' => ['/var/log/messages', '/var/log/*log'], 'fields' => { 'type' => 'syslog' } }] end ## KIBANA diff --git a/test/cookbooks/ellktest/templates/default/logstash/logstash.conf.erb b/test/cookbooks/ellktest/templates/default/logstash/logstash.conf.erb index c4208e9..6e56ef2 100644 --- a/test/cookbooks/ellktest/templates/default/logstash/logstash.conf.erb +++ b/test/cookbooks/ellktest/templates/default/logstash/logstash.conf.erb @@ -11,22 +11,31 @@ input { } filter { - if [type] == "syslog" { + if [type] == "syslog" { grok { - match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } - add_field => [ "received_at", "%{@timestamp}" ] - add_field => [ "received_from", "%{host}" ] + overwrite => "message" + match => [ + "message", + "%{SYSLOGTIMESTAMP:timestamp} %{IPORHOST:host} (?:%{PROG:program}(?:\[%{POSINT:pid}\])?: )?%{GREEDYDATA:message}" + ] } syslog_pri { } date { - match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] + # season to taste for your own syslog format(s) + match => [ + "timestamp", + "MMM d HH:mm:ss", + "MMM dd HH:mm:ss", + "ISO8601" + ] } } } + # TODO: logstash doesn't convert 1:1 with ruby or json, hash passing will have to wait # for now we consider all elasticsearch nodes have a logstash service on the same machine output { elasticsearch { host => localhost} - stdout { codec => rubydebug } -} + stdout { codec => json } +} \ No newline at end of file