New provision code with Ansible

.gitignore

@@ -39,5 +39,7 @@ coverage.txt
 .golangci.yml
 config.yml
 deploy/aws/config_files/*
+deploy/aws/provision/config_files/*
 !deploy/aws/config_files/README.md
 deploy/aws/modules/*/data
+

deploy/aws/provision.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+export TF_STATE=/tmp/terraform.tfstate
+
+if [ ! -f $GOBIN/terraform-inventory ];
+then
+    echo -e "Please install terraform-invenotry in your GOBIN \ngo get github.com/adammck/terraform-inventor"
+    exit 1
+fi
+
+echo "Pulling terraform remote state to: $TF_STATE"
+terraform state pull > $TF_STATE
+
+if [ -z "$@" ];
+then
+    ansible-playbook --inventory-file=$GOBIN/terraform-inventory --private-key ~/.ssh/id_rsa_free provision/main.yml -f 10
+else
+    echo "Using custom Ansible Playbook command."
+    ansible-playbook $@
+fi

deploy/aws/provision/ansible.cfg

@@ -0,0 +1,15 @@
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+[defaults]
+host_key_checking = False

deploy/aws/provision/main.yml

@@ -0,0 +1,71 @@
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+---
+- hosts: localhost
+  gather_facts: false
+  connection: local
+  vars_files:
+    - vars.yml
+  environment:
+    - GOOS: linux
+    - GOARCH: amd64
+    - GO111MODULE: 'on'
+  tasks:
+    - include: tasks/common/build.yml
+    - include: tasks/common/config.yml
+
+- hosts: type_aws_instance
+  remote_user: ec2-user
+  become: true
+  gather_facts: false
+  vars_files:
+    - vars.yml
+  tasks:
+    - include: tasks/common/main.yml
+
+- hosts: qed-server
+  remote_user: ec2-user
+  become: true
+  gather_facts: false
+  vars_files:
+    - vars.yml
+  tasks:
+    - include: tasks/qed/main.yml
+
+- hosts: riot
+  remote_user: ec2-user
+  become: true
+  gather_facts: false
+  vars_files:
+    - vars.yml
+  tasks:
+    - include: tasks/riot/main.yml
+
+- hosts: inmemory-storage
+  remote_user: ec2-user
+  become: true
+  gather_facts: false
+  vars_files:
+    - vars.yml
+  tasks:
+    - include: tasks/inmemory_storage/main.yml
+
+- hosts: prometheus
+  remote_user: ec2-user
+  become: true
+  gather_facts: false
+  vars_files:
+    - vars.yml
+  tasks:
+    - include: tasks/prometheus/main.yml

deploy/aws/provision/tasks/common/build.yml

@@ -0,0 +1,26 @@
+---
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+- name: Create BIN dir
+  file:
+      path: config_files/bin
+      state: directory
+
+- name: Build QED, Inmemory Storage and Riot
+  shell: > 
+    go build -o {{ item }}
+  with_items:
+    - config_files/bin/qed ../../../
+    - config_files/bin/storage ../../../tests/gossip/test_service.go
+    - config_files/bin/riot ../../../tests/riot.go

deploy/aws/provision/tasks/common/main.yml

@@ -0,0 +1,46 @@
+---
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+- name: Create QED and Node Exporter install dir
+  file:
+      path: /var/qed
+      state: directory
+
+- name: Install additional packages
+  yum:
+    name: "{{ packages }}"
+  vars:
+    packages:
+    - awslogs
+
+- name: Download Node Exporter
+  unarchive:
+    src: https://github.com/prometheus/node_exporter/releases/download/v0.17.0/node_exporter-0.17.0.linux-amd64.tar.gz
+    dest: /var/qed
+    remote_src: yes
+
+- name: Create Node Exporter link 
+  file:
+    src: /var/qed/node_exporter-0.17.0.linux-amd64
+    dest: /var/qed/node_exporter
+    state: link
+
+- name: Copy certs to remote
+  copy:
+      dest: /var/qed/{{ item }}
+      src: config_files/{{ item }}
+  with_items:
+    - server.crt
+    - server.key
+    - id_ed25519

deploy/aws/provision/tasks/inmemory_storage/main.yml

@@ -0,0 +1,21 @@
+---
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+- name: Copy Inmemory Storage Binary
+  copy:
+      dest: /var/qed/{{ item }}
+      src: config_files/bin/{{ item }}
+      mode: 0777
+  with_items:
+    - storage

deploy/aws/provision/tasks/prometheus/main.yml

@@ -0,0 +1,50 @@
+---
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+- name: Download Prometheus
+  unarchive:
+    src: https://github.com/prometheus/prometheus/releases/download/v2.7.0/prometheus-2.7.0.linux-amd64.tar.gz
+    dest: /var
+    remote_src: yes
+
+- name: Create Node Exporter link 
+  file:
+    src: /var/prometheus-2.7.0.linux-amd64
+    dest: /var/prometheus
+    state: link
+
+- name: Install Grafana
+  yum:
+      update_cache: yes 
+      name: https://dl.grafana.com/oss/release/grafana-5.4.2-1.x86_64.rpm
+
+- name: Copy Grafana config and dashboards to remote
+  copy:
+      dest: /etc/grafana/provisioning/{{ item }}
+      src: ../modules/prometheus/data/provisioning/{{ item }}
+      owner: root
+      group: grafana
+  with_items:
+    - dashboards
+    - datasources
+  register: grafana_config
+
+- name: Restart Grafana
+  service:
+    name: grafana-server
+    state: restarted
+  when: grafana_config.changed
+

deploy/aws/provision/tasks/qed/main.yml

@@ -0,0 +1,21 @@
+---
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+- name: Copy QED Binary
+  copy:
+      dest: /var/qed/{{ item }}
+      src: config_files/bin/{{ item }}
+      mode: 0777
+  with_items:
+    - qed

deploy/aws/provision/tasks/riot/main.yml

@@ -0,0 +1,21 @@
+---
+#  Copyright 2018 Banco Bilbao Vizcaya Argentaria, S.A.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+- name: Copy Riot Binary
+  copy:
+      dest: /var/qed/{{ item }}
+      src: config_files/bin/{{ item }}
+      mode: 0777
+  with_items:
+    - riot

deploy/aws/provision/vars.yml

@@ -0,0 +1 @@
+---

go.mod

@@ -2,6 +2,7 @@ module github.com/bbva/qed
 
 require (
 	github.com/VictoriaMetrics/fastcache v1.3.0
+	github.com/adammck/terraform-inventory v0.6.1 // indirect
 	github.com/bbva/raft-badger v0.1.1
 	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect

go.sum

@@ -7,6 +7,8 @@ github.com/VictoriaMetrics/fastcache v1.0.2 h1:dj179vXczNtKDLjmiOHRasMMv437rPpUt
 github.com/VictoriaMetrics/fastcache v1.0.2/go.mod h1:4zLf+tCNHtaJzQfIkJ+vBXS1O98gXA/KbKtUv/W1Tdo=
 github.com/VictoriaMetrics/fastcache v1.3.0 h1:6QIxfjt2tCFH+rqAgoOIuYvm230IwLwiLyxoocTcHIc=
 github.com/VictoriaMetrics/fastcache v1.3.0/go.mod h1:CH4+tdFBMyTCU9HV/ifYqJvn7ECWfK1E7ES6eWt6F2o=
+github.com/adammck/terraform-inventory v0.6.1 h1:rlUJV0ooEYDYr/RFBjlXwIz2EVH9BeGlcxlk26wYw4s=
+github.com/adammck/terraform-inventory v0.6.1/go.mod h1:ARrwFGZ06YoSbxqJ1RlwljUYFM8fo4edCA4Tiq3Nvtw=
 github.com/allegro/bigcache v1.1.0 h1:MLuIKTjdxDc+qsG2rhjsYjsHQC5LUGjIWzutg7M+W68=
 github.com/allegro/bigcache v1.1.0/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA=