refactor: deploy process to use systemd

Now all the services use systemd units instead running raw start/stop scripts.
Also now all the logs goes to /var/log/messages. Only this log is appended to AWS CloudWatch.

.dockerignore

@@ -2,6 +2,7 @@
 **qed
 storage
 !storage
+**deploy
 **riot
 **prometheus
 **/.git

deploy/aws/main.tf

@@ -67,7 +67,7 @@ module "inmemory-storage" {
 
 module "agent-publisher" {
   source = "./modules/agent"
-  count  = 2
+  count  = 1
 
   role                   = "publisher"
   name                   = "agent-publisher"
@@ -82,7 +82,7 @@ module "agent-publisher" {
 
 module "agent-monitor" {
   source = "./modules/agent"
-  count  = 2
+  count  = 1
 
   role                   = "monitor"
   name                   = "agent-monitor"
@@ -97,7 +97,7 @@ module "agent-monitor" {
 
 module "agent-auditor" {
   source = "./modules/agent"
-  count  = 2
+  count  = 1
 
   role                   = "auditor"
   name                   = "agent-auditor"

deploy/aws/provision/main.yml

@@ -103,18 +103,19 @@
 
 - hosts: role_qed
   remote_user: ec2-user
+  gather_facts: true
   serial: 1
   become: true
   vars_files:
     - vars.yml
   tasks:
     - name: Restart all services
-      shell: >
-        {{ item }}
+      systemd:
+        name: "{{ item }}"
+        state: restarted
       with_items:
-        - /var/qed/qed-stop.sh
-        - /var/qed/qed-start.sh
-        - /var/qed/exporter-start.sh
+        - qed
+        - exporter
     - name: wait for raised api port
       wait_for:
         port: 8800
@@ -138,12 +139,12 @@
     - vars.yml
   tasks:
     - name: Restart all services
-      shell: >
-        {{ item }}
+      systemd:
+        name: "{{ item }}"
+        state: restarted
       with_items:
-        - /var/qed/qed-stop.sh
-        - /var/qed/qed-start.sh
-        - /var/qed/exporter-start.sh
+        - qed
+        - exporter
   tags:
     - start-agents
 
@@ -154,11 +155,12 @@
     - vars.yml
   tasks:
     - name: Start all services
-      shell: > 
-        {{ item }}
+      systemd:
+        name: "{{ item }}"
+        state: started
       with_items:
-        - /var/qed/storage-start.sh
-        - /var/qed/exporter-start.sh
+        - storage
+        - exporter
   tags:
     - start-storage
 
@@ -168,12 +170,13 @@
   vars_files:
     - vars.yml
   tasks:
-    - name: Restart riot
-      shell: >
-        {{ item }}
+    - name: Restart all services
+      systemd:
+        name: "{{ item }}"
+        state: restarted
       with_items:
-        - /var/qed/riot-stop.sh
-        - /var/qed/riot-start.sh
+        - riot
+        - exporter
     - name: wait for raised riot port
       wait_for:
         port: 7700
@@ -188,10 +191,12 @@
     - vars.yml
   tasks:
     - name: Start all services
-      shell: > 
-        {{ item }}
+      systemd:
+        name: "{{ item }}"
+        state: started
       with_items:
-        - /var/qed/exporter-start.sh
+        - prometheus
+        - exporter
   tags:
     - start-prometheus
     - prometheus

deploy/aws/provision/tasks/common/main.yml

@@ -111,9 +111,36 @@
     mode: 0775
   with_items:
     - start.sh
+    - stop.sh
+
+- name: Create Node Exporter start|stop script
+  template:
+    dest: /var/qed/{{ item }} 
+    src: ../../templates/{{ item }}.j2
+    force: true
+    mode: 0775
+  with_items:
+    - qed-start.sh
+    - qed-stop.sh
+
+- name: Install Node Exporter systemd service unit file
+  template:
+    dest: /etc/systemd/system/{{ item }}
+    src: ../../templates/{{ item }}.j2
+  with_items:
+    - exporter.service
+  register: exporter_unit
 
 - name: Create QED tmp dir link
   file:
     src: /var/qed
     dest: /var/tmp/qed
     state: link
+
+- name: Restart Node Exporter
+  service:
+    name: exporter
+    state: restarted
+    daemon_reload: yes
+  when: exporter_unit.changed
+

deploy/aws/provision/tasks/inmemory_storage/main.yml

@@ -27,4 +27,27 @@
     force: true
     mode: 0775
   with_items:
-    - storage-start.sh
+    - storage-start.sh
+    - storage-stop.sh
+
+- name: Install Storage systemd service unit file
+  template:
+    dest: /etc/systemd/system/{{ item }}
+    src: ../../templates/{{ item }}.j2
+  with_items:
+    - storage.service
+  register: storage_unit
+
+- name: Enable and run Storage service
+  systemd:
+    name: storage
+    state: started
+    enabled: yes
+    daemon_reload: yes
+
+- name: Restart Storage
+  service:
+    name: storage
+    state: restarted
+    daemon_reload: yes
+  when: storage_unit.changed

deploy/aws/provision/tasks/qed/main.yml

@@ -38,4 +38,25 @@
     - qed-start.sh
     - qed-stop.sh
 
+- name: Install QED systemd service unit file
+  template:
+    dest: /etc/systemd/system/{{ item }}
+    src: ../../templates/{{ item }}.j2
+  with_items:
+    - qed.service
+  register: qed_unit
+
+- name: Enable and run QED service
+  systemd:
+    name: qed
+    state: started
+    enabled: yes
+    daemon_reload: yes
+
+- name: Restart Qed
+  service:
+    name: qed
+    state: restarted
+    daemon_reload: yes
+  when: qed_unit.changed
 

deploy/aws/provision/tasks/riot/main.yml

@@ -20,7 +20,7 @@
   with_items:
     - riot
 
-- name: Create QED start|stop script
+- name: Create Riot start|stop script
   template:
     dest: /var/qed/{{ item }}
     src: ../../templates/{{ item }}.j2
@@ -29,3 +29,25 @@
   with_items:
     - riot-start.sh
     - riot-stop.sh
+
+- name: Install Riot systemd service unit file
+  template:
+    dest: /etc/systemd/system/{{ item }}
+    src: ../../templates/{{ item }}.j2
+  with_items:
+    - riot.service
+  register: riot_unit
+
+- name: Enable and run Riot service
+  systemd:
+    name: riot
+    state: started
+    enabled: yes
+    daemon_reload: yes
+
+- name: Restart Riot
+  service:
+    name: riot
+    state: restarted
+    daemon_reload: yes
+  when: riot_unit.changed

deploy/aws/provision/templates/awslogs.conf.j2

@@ -140,13 +140,3 @@ log_stream_name = {{ hostvars[inventory_hostname]['tags.Name'] }}
 {% endif %}
 initial_position = start_of_file
 log_group_name = /var/log/messages
-
-[/var/tmp/qed]
-datetime_format = %b %d %H:%M:%S
-file = /var/qed/qed.log
-buffer_duration = 5000
-{% if 'type_aws_instance' in group_names %}
-log_stream_name = {{ hostvars[inventory_hostname]['tags.Name'] }}
-{% endif %}
-initial_position = start_of_file
-log_group_name = qed

deploy/aws/provision/templates/exporter-start.sh.j2

@@ -16,4 +16,4 @@
 #!/bin/bash
 
 export NODE_EXPORTER_HOME=/var/qed/node_exporter
-nohup $NODE_EXPORTER_HOME/node_exporter &
+$NODE_EXPORTER_HOME/node_exporter

deploy/aws/provision/templates/exporter-stop.sh.j2

@@ -0,0 +1,18 @@
+{#
+ Copyright 2018-2019 Banco Bilbao Vizcaya Argentaria, S.A.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+#}
+#!/bin/bash
+
+killall node_exporter || true

deploy/aws/provision/templates/exporter.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=Node exporter
+Documentation=https://github.com/bbva/qed
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=root
+Group=root
+Restart=always
+RestartSec=5
+WorkingDirectory=/var/qed
+RuntimeDirectory=root
+RuntimeDirectoryMode=0750
+ExecStart=/var/qed/exporter-start.sh
+ExecStop=/var/qed/exporter-stop.sh
+LimitNOFILE=10000
+TimeoutStopSec=20
+
+[Install]
+WantedBy=multi-user.target

deploy/aws/provision/templates/qed-start.sh.j2

@@ -19,14 +19,14 @@ ulimit -n 819200
 
 export QED_HOME=/var/qed
 {% if 'role_qed' in group_names %}
-nohup $QED_HOME/qed start >> $QED_HOME/qed.log 2>&1 &
+$QED_HOME/qed start
 {% endif %}
 {% if 'role_monitor' in group_names %}
-nohup $QED_HOME/qed agent monitor >> $QED_HOME/qed.log 2>&1 &
+$QED_HOME/qed agent monitor
 {% endif %}
 {% if 'role_auditor' in group_names %}
-nohup $QED_HOME/qed agent auditor >> $QED_HOME/qed.log 2>&1 &
+$QED_HOME/qed agent auditor
 {% endif %}
 {% if 'role_publisher' in group_names %}
-nohup $QED_HOME/qed agent publisher >> $QED_HOME/qed.log 2>&1 &
+$QED_HOME/qed agent publisher
 {% endif %}

deploy/aws/provision/templates/qed.service.j2

@@ -0,0 +1,21 @@
+[Unit]
+Description=QED service
+Documentation=https://github.com/bbva/qed
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=root
+Group=root
+Restart=always
+RestartSec=10
+WorkingDirectory=/var/qed
+RuntimeDirectory=root
+RuntimeDirectoryMode=0750
+ExecStart=/var/qed/qed-start.sh
+ExecStop=/var/qed/qed-stop.sh
+LimitNOFILE=10000
+TimeoutStopSec=20
+
+[Install]
+WantedBy=multi-user.target

deploy/aws/provision/templates/riot-start.sh.j2

@@ -25,6 +25,5 @@ nohup $QED_HOME/riot --api \
     --apikey terraform_qed \
     --n ${reqs} \
 {% for host in groups['name_qed-0'] %}
-    --endpoint "{{ hostvars[host]['ansible_eth0']['ipv4']['address'] }}:8400" \
+    --endpoint "http://{{ hostvars[host]['ansible_eth0']['ipv4']['address'] }}:8800"
 {% endfor %}
-    >> $QED_HOME/qed.log 2>&1 &

deploy/aws/provision/templates/riot.service.j2

@@ -0,0 +1,20 @@
+[Unit]
+Description=Riot benchmark tool
+Documentation=https://github.com/bbva/qed
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+User=root
+Group=root
+Restart=on-failure
+WorkingDirectory=/var/qed
+RuntimeDirectory=root
+RuntimeDirectoryMode=0750
+ExecStart=/var/qed/riot-start.sh
+ExecStop=/var/qed/riot-stop.sh
+LimitNOFILE=10000
+TimeoutStopSec=20
+
+[Install]
+WantedBy=multi-user.target

deploy/aws/provision/templates/storage-start.sh.j2

@@ -16,5 +16,5 @@
 #!/bin/bash
 
 export STORAGE_HOME=/var/qed
-nohup $STORAGE_HOME/storage start >> $STORAGE_HOME/qed.log 2>&1 &
+nohup $STORAGE_HOME/storage start