Merge pull request #1 from leifmadsen/master

Merge effort to automatically deploy ELK stack

.gitignore

@@ -0,0 +1,3 @@
+*.retry
+roles/
+inventory

README.md

@@ -1,2 +1,50 @@
-# ansible-elk-stack
-Deploy an ELK stack with Ansible
+# ansible-cira
+
+Deploy a continuous integration reference architecture with Jenkins to test
+OpenStack with [TripleO
+Quickstart](https://github.com/openstack/tripleo-quickstart).
+
+
+## Requirements
+
+You'll need to install the `shade` dependency so that you can interact with
+OpenStack (assuming you are deploying to an OpenStack cloud).
+
+    pip install --user shade
+
+For Ansible, several roles are required, and you can install them as follows:
+
+    ansible-galaxy install -r requirements.yml
+
+## Setup OpenStack Connection
+
+If you're going to install to an OpenStack cloud, you'll need to configure a
+cloud to connect to. You can do this by creating the `~/.config/openstack/`
+directory and placing the following contents into the `cloud.yml` file within
+that directory (adjust to your own cloud connection):
+
+    clouds:
+        mycloud:
+            auth:
+                auth_url: http://theclowd.com:5000/v2.0
+                username: cloud_user
+                password: cloud_pass
+                project_name: "My Cloud Project"
+
+## Deployment
+
+You may need to adjust the `host_vars/localhost` file to adjust the
+`security_group` names, as the playbook does not currently create security
+groups and rules for you. It is assumed you've created the following sets of
+security groups, and opened the corresponding ports:
+
+* elasticsearch
+  * `TCP: 9200`
+* filebeat-input
+  * `TCP: 5440`
+* web_ports
+  * `TCP: 80, 443`
+
+After configuration, you can run the following comment which will connect to
+localhost to run the `shade` applications, authenticate to the OpenStack API
+you've supplied in `cloud.yml` and then deploy the stack.

ansible.cfg

@@ -0,0 +1,8 @@
+[defaults]
+roles_path = ./roles
+gathering = smart
+fact_caching = jsonfile
+fact_caching_connection = ~/.ansible/cachedir
+fact_caching_timeout = 86400
+host_key_checking = False
+inventory = ./inventory

files/filebeat.yml

@@ -0,0 +1,21 @@
+filebeat:
+  prospectors:
+    -
+      paths:
+        - /home/jenkins/jenkins-agent/workspace/tripleo-artifact-deploy-test/job_*.log
+      input_type: log
+      document_type: "ansible"
+      include_lines: [ '(^PLAY RECAP.+)', '(^TASK.+)', '(^task path.+)', '(Monday)|(Tuesday)|(Wednesday)|(Thursday)|(Friday)|(Saturday)|(Sunday)', '(^ok\:.+)', '(^skipping\:.+)', '(^failed\:.+)', '(^included\:.+)', '(^changed\:.+)' ]
+  registry_file: /var/lib/filebeat/registry
+output:
+  logstash:
+    hosts: ["10.10.0.1:5044"]
+shipper:
+logging:
+  level: debug
+  to_files: true
+  to_syslog: false
+  files:
+    path: /var/log/mybeat
+    name: mybeat.log
+    keepfiles: 7

host_vars/localhost

@@ -0,0 +1,6 @@
+use_openstack_deploy: true
+
+instance_list:
+  - { name: elasticsearch, security_groups: "default,elasticsearch" }
+  - { name: logstash, security_groups: "default,filebeat-input" }
+  - { name: kibana, security_groups: "default,web_ports" }

inventory

@@ -0,0 +1 @@
+localhost   ansible_connection=local

openstack.yml

@@ -0,0 +1,49 @@
+# vim: set ft=ansible
+- name: Deploy on OpenStack
+  hosts: localhost
+  gather_facts: false
+  vars_files:
+    - ~/.ansible/vars/elk_vars.yml
+
+  tasks:
+    - name: Deploy an instance
+      os_server:
+        name: "{{ cloud_name_prefix }}-{{ item.name }}"
+        state: present
+        cloud: "{{ cloud_name }}"
+        region_name: "{{ cloud_region_name }}"
+        availability_zone: "{{ cloud_availability_zone }}"
+        image: "{{ cloud_image }}"
+        flavor: "{{ cloud_flavor }}"
+        key_name: "{{ cloud_key_name }}"
+        boot_from_volume: true
+        terminate_volume: true
+        volume_size: 20
+        security_groups: "{{ item.security_groups }}"
+        auto_ip: yes
+        timeout: 200
+      register: instances
+      with_items: "{{ instance_list }}"
+
+    - debug:
+        var: instances
+        verbosity: 1
+
+    - name: Validate the host is available
+      command: >
+        ssh -o BatchMode=yes -o StrictHostKeyChecking=no
+        centos@{{ item.server.public_v4 }}
+      register: result
+      until: result|success
+      retries: 300
+      delay: 5
+      with_items: "{{ instances.results }}"
+
+    - name: Add host to inventory
+      add_host:
+        name: "{{ item.item.name }}"
+        groups: "{{ item.item.name }}"
+        ansible_host: "{{ item.server.public_v4 }}"
+        ansible_user: centos
+        ansible_become: true
+      with_items: "{{ instances.results }}"

requirements.yml

@@ -0,0 +1,10 @@
+---
+- src: geerlingguy.repo-epel
+- src: https://github.com/leifmadsen/ansible-role-elasticsearch.git
+  version: master
+  name: leifmadsen.elasticsearch
+- src: https://github.com/leifmadsen/ansible-role-logstash.git
+  version: master
+  name: leifmadsen.logstash
+- src: geerlingguy.nginx
+- src: leifmadsen.kibana-4

site.yml

@@ -0,0 +1,46 @@
+# main.yml
+---
+
+# Deploy to OpenStack Servers
+- include: openstack.yml
+  when: use_openstack_deploy
+
+- hosts: elasticsearch
+  vars_files:
+    - vars/main.yml
+
+  roles:
+    - { role: 'geerlingguy.repo-epel' }
+    - { role: 'leifmadsen.elasticsearch' }
+
+- hosts: logstash
+  vars_files:
+    - vars/main.yml
+
+  roles:
+    - { role: 'geerlingguy.repo-epel' }
+    - { role: 'leifmadsen.logstash' }
+
+- hosts: kibana
+  vars_files:
+    - vars/main.yml
+  roles:
+    - { role: 'geerlingguy.repo-epel' }
+    - { role: 'geerlingguy.nginx' }
+    - { role: 'leifmadsen.kibana-4' }
+
+  post_tasks:
+    - name: Validate SELinux is enabled
+      selinux:
+        policy: targeted
+        state: enforcing
+
+    - name: SELinux -- Enable httpd_can_network_connect
+      seboolean:
+        name: httpd_can_network_connect
+        state: yes
+        persistent: yes
+
+    - name: Where is Kibana located?
+      debug:
+        msg: "Kibana can be reached at http://{{hostvars['kibana']['ansible_host']}}"

templates/03-beats.conf.j2

@@ -0,0 +1,32 @@
+input {
+  beats {
+    port => 5044
+    codec => multiline {
+      pattern => "(^task path.+)|(^%{DAY}.+)|(^ok:.+)|(^skipping:.+)|(^failed:.+)|(^included:.+)|(^changed:.+)|(^fatal:.+)"
+      what => "previous"
+      auto_flush_interval => 3600
+    }
+  }
+}
+
+filter {
+    mutate {
+      gsub => ['message', "\n", " "]
+    }
+
+    grok {
+      match => [ "message", "%{WORD}\s\[(?<task_name>.*?)\]\s+(?:[*]+)?(?:\:\s+)?(?:\s.*?%{UNIXPATH:task_path})?\s+(?<task_date>%{DAY} [\d]+ [\w]+ [\d]+)\s+%{TIME:task_time}\s+(?<tz_offset>\+\d+)\s+\((?<task_execution_time>.*?)\)\s+%{TIME:total_execution_time}(?:\s+[*]+\s+)?(?:(?<task_result>\w+)\:\s+\[(?<task_location>.*?)\](?:\:\s+)?(?:\s\=\>\s)?(%{GREEDYDATA:task_data}))?" ]
+      add_tag => [ "grokked" ]
+    }
+}
+
+output {
+#  stdout {
+#    codec => line { format => "%{message}" }
+#    codec => rubydebug 
+#  }
+
+  elasticsearch {
+    hosts => ["{{ elasticsearch_network_address }}:{{ elasticsearch_http_port }}"]
+  }
+}

vars/main.yml

@@ -0,0 +1,12 @@
+# Variables setup for use in site.yml run.
+# Use host_vars/ or group_vars/ for openstack.yml setup.
+
+logstash_configuration_files:
+  - 03-beats.conf
+
+elasticsearch_script_inline: false
+elasticsearch_script_index: false
+elasticsearch_http_port: 9200
+elasticsearch_network_address: "{{ hostvars['elasticsearch']['ansible_eth0']['ipv4']['address'] }}"
+elasticsearch_url: http://{{ elasticsearch_network_address }}:{{ elasticsearch_http_port }}
+elasticsearch_network_host: _eth0:ipv4