[BACKPORT] Upgrade Zookeeper to v3.5.8

CHANGELOG-0.7.md

@@ -9,6 +9,7 @@
 ### Updated
 
 - [#1965](https://github.com/epiphany-platform/epiphany/issues/1965) - Upgrade Elasticsearch Curator to v5.8.3
+- [#1927](https://github.com/epiphany-platform/epiphany/issues/1927) - Upgrade Zookeeper to v3.5.8
 
 ## [0.7.2] 2020-10-07
 

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt

@@ -153,7 +153,7 @@ https://github.com/danielqsj/kafka_exporter/releases/download/v1.2.0/kafka_expor
 https://github.com/prometheus/node_exporter/releases/download/v0.16.0/node_exporter-0.16.0.linux-amd64.tar.gz
 https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz
 https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz
-https://archive.apache.org/dist/zookeeper/zookeeper-3.4.12/zookeeper-3.4.12.tar.gz
+https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
 https://archive.apache.org/dist/ignite/2.7.6/apache-ignite-2.7.6-bin.zip
 https://releases.hashicorp.com/vault/1.4.0/vault_1.4.0_linux_amd64.zip
 https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt

@@ -150,7 +150,7 @@ https://github.com/danielqsj/kafka_exporter/releases/download/v1.2.0/kafka_expor
 https://github.com/prometheus/node_exporter/releases/download/v0.16.0/node_exporter-0.16.0.linux-amd64.tar.gz
 https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz
 https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz
-https://archive.apache.org/dist/zookeeper/zookeeper-3.4.12/zookeeper-3.4.12.tar.gz
+https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
 https://archive.apache.org/dist/ignite/2.7.6/apache-ignite-2.7.6-bin.zip
 https://releases.hashicorp.com/vault/1.4.0/vault_1.4.0_linux_amd64.zip
 https://get.helm.sh/helm-v3.2.0-linux-amd64.tar.gz

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt

@@ -169,7 +169,7 @@ kubernetes-cni 0.8.6-00
 [files]
 https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar
 https://archive.apache.org/dist/kafka/2.3.1/kafka_2.12-2.3.1.tgz
-https://archive.apache.org/dist/zookeeper/zookeeper-3.4.12/zookeeper-3.4.12.tar.gz
+https://archive.apache.org/dist/zookeeper/zookeeper-3.5.8/apache-zookeeper-3.5.8-bin.tar.gz
 https://github.com/danielqsj/kafka_exporter/releases/download/v1.2.0/kafka_exporter-1.2.0.linux-amd64.tar.gz
 https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz
 https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/zookeeper.yml

@@ -0,0 +1,50 @@
+---
+- name: Include defaults from zookeeper role
+  include_vars:
+    file: roles/zookeeper/defaults/main.yml
+    name: zookeeper_defaults
+
+- name: Include pre-flight checks
+  include_tasks: zookeeper/preflight-check.yml
+
+- name: Get installed Zookeeper's version
+  stat:
+    path: /opt/zookeeper
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: linked_zookeeper
+
+- name: Set installed Zookeeper version as fact
+  set_fact:
+    before_upgrade_zookeeper_version: "{{ linked_zookeeper.stat.lnk_target | regex_search('\\d+\\.\\d+\\.\\d+') }}"
+
+- name: Check for upgrade flag file
+  stat:
+    path: "{{ lock_file }}"
+    get_attributes: false
+    get_checksum: false
+    get_mime: false
+  register: lock_file_status
+
+- name: Upgrade Zookeeper if newer version is available
+  block:
+    - name: Create upgrade flag file
+      file:
+        path: "{{ lock_file }}"
+        state: touch
+
+    - name: Stop Zookeeper service
+      service:
+        name: zookeeper
+        state: stopped
+
+    - name: Include upgrade Zookeeper task
+      include_tasks: zookeeper/install-upgrade.yml
+  when:
+    - lock_file_status.stat.exists or before_upgrade_zookeeper_version is version( zookeeper_defaults.zookeeper_version, '<' )
+
+- name: Remove Zookeeper upgrade flag file
+  file:
+    path: "{{ lock_file }}"
+    state: absent

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/zookeeper/install-upgrade.yml

@@ -0,0 +1,79 @@
+---
+
+# Some tasks in this file are the same as in zookeeper role. It should be refactored (with splitting code into separate files) in order to reuse common tasks here.
+- name: Download Zookeeper binaries
+  include_role:
+    name: download
+    tasks_from: download_file
+  vars:
+    file_name: "{{ zookeeper_defaults.zookeeper_bin_filename }}"
+
+- name: Create Zookeeper directories
+  become: yes
+  file:
+    path: "/opt/zookeeper-{{ zookeeper_defaults.zookeeper_version }}"
+    recurse: yes
+    owner: "{{ zookeeper_defaults.zookeeper_user }}"
+    group: "{{ zookeeper_defaults.zookeeper_group }}"
+    mode: u=rwx,g=rx,o=rx
+    state: directory
+
+- name: Uncompress Zookeeper installation file
+  unarchive:
+    remote_src: yes
+    src: "{{ download_directory }}/{{ zookeeper_defaults.zookeeper_bin_filename }}"
+    dest: /opt/zookeeper-{{ zookeeper_defaults.zookeeper_version }}
+    creates: "/opt/zookeeper-{{ zookeeper_defaults.zookeeper_version }}/bin"
+    extra_opts: [--strip-components=1]
+    mode: u=rwx,g=rx,o=rx
+    owner: "{{ zookeeper_defaults.zookeeper_user }}"
+    group: "{{ zookeeper_defaults.zookeeper_group }}"
+
+- name: Copy configuration from previous version
+  copy:
+    remote_src: yes
+    src: /opt/zookeeper/conf/
+    dest: /opt/zookeeper-{{ zookeeper_defaults.zookeeper_version }}/conf
+    mode: preserve
+
+- name: Link /opt/zookeeper to recently installed version
+  file:
+    dest: /opt/zookeeper
+    state: link
+    src: /opt/zookeeper-{{ zookeeper_defaults.zookeeper_version }}
+    force: yes
+
+- name: Reconfigure Zookeeper service to use symbolic link
+  lineinfile:
+    path: /lib/systemd/system/zookeeper.service
+    state: present
+    regexp: '^ExecStart=/opt/zookeeper-.*'
+    line: "ExecStart=/opt/zookeeper/bin/zkServer.sh start-foreground"
+
+- name: Check if any snapshots exists in data dir what is necessary in order to run zookeeper after upgrade
+  find:
+    paths: "{{ zookeeper_defaults.zookeeper_data_dir }}/version-2"
+    patterns: "snapshot.*"
+  register: snapshot_exists
+
+# From 3.5.5 version, ZooKeeper is not able to start when no snapshot files present, what is valid scenario in 3.4.X version. Empty snapshot downloaded from Zookeeper's Jira ticket. 
+- name: Copy empty snapshot if not exists
+  copy:
+    dest: "{{ zookeeper_defaults.zookeeper_data_dir }}/version-2" 
+    src: roles/zookeeper/files/snapshot.0
+    mode: u=rw,g=r,o=r
+    owner: "{{ zookeeper_defaults.zookeeper_user }}"
+    group: "{{ zookeeper_defaults.zookeeper_group }}"
+  when: snapshot_exists.matched == 0
+
+- name: Start Zookeeper service
+  systemd:
+    name: zookeeper
+    state: started
+    daemon-reload: yes
+
+- name: Remove previous version binaries
+  file:
+    path: /opt/zookeeper-{{ before_upgrade_zookeeper_version }}
+    state: absent
+  when: before_upgrade_zookeeper_version != zookeeper_defaults.zookeeper_version

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/zookeeper/preflight-check.yml

@@ -0,0 +1,14 @@
+---
+  - name: Check if Zookeeper is installed in default location
+    stat:
+      path: /opt/zookeeper/bin/zkServer.sh
+      get_attributes: false
+      get_checksum: false
+      get_mime: false
+    register: zookeeper_exec_file
+
+  - name: Assert Zookeeper location
+    assert:
+      that:
+        - zookeeper_exec_file.stat.exists
+      fail_msg: Zookeeper not found in /opt/zookeeper (Epiphany default) - check your configuration

core/src/epicli/data/common/ansible/playbooks/roles/zookeeper/defaults/main.yml

@@ -1,5 +1,8 @@
 ---
 
+zookeeper_version: 3.5.8
+zookeeper_bin_filename: "apache-zookeeper-3.5.8-bin.tar.gz"
+
 zookeeper_hosts: "{{ groups['zookeeper'] }}"
 
 zookeeper_data_dir: /var/lib/zookeeper

core/src/epicli/data/common/ansible/playbooks/roles/zookeeper/tasks/main.yml

@@ -1,5 +1,4 @@
 ---
-# Zookeeper tasks - resides on masters
 
 - name: Create Zookeeper group
   group:
@@ -30,19 +29,18 @@
     path: "{{ prometheus_jmx_path }}"
   register: exporter
 
-- name: Set zookeeper variable with version name
+- name: Set Zookeeper variable with version name
   set_fact:
-    zookeeper_name: "zookeeper-{{ specification.version }}"
+    zookeeper_name: "zookeeper-{{ zookeeper_version }}"
   changed_when: false
 
-- name: Set zookeeper install dir for {{ zookeeper_name }}
+- name: Set Zookeeper install dir for {{ zookeeper_name }}
   set_fact:
     zookeeper_install_dir: "/opt/{{ zookeeper_name }}"
-  changed_when: false
 
 - name: Set Zookeeper file name to install
   set_fact:
-    zookeeper_file_name: "{{ specification.file_name }}"
+    zookeeper_file_name: "{{ zookeeper_bin_filename }}"
 
 - name: Download Zookeeper binaries
   include_role:
@@ -58,7 +56,7 @@
     recurse: yes
     owner: "{{ zookeeper_user }}"
     group: "{{ zookeeper_group }}"
-    mode: 0750
+    mode: u=rwx,g=rx,o=
     state: directory
   with_items:
   - "{{ zookeeper_install_dir }}"
@@ -71,7 +69,7 @@
     dest: "{{ zookeeper_install_dir }}"
     creates: "{{ zookeeper_install_dir }}/bin"
     extra_opts: [--strip-components=1]
-    mode: 0755
+    mode: u=rwx,g=rx,o=rx
     owner: "{{ zookeeper_user }}"
     group: "{{ zookeeper_group }}"
   check_mode: false
@@ -88,7 +86,7 @@
     - "{{ zookeeper_install_dir }}/conf"
     - "/etc/zookeeper/conf"
 
-- name: Create zookeeper service
+- name: Create Zookeeper service
   template:
     src: zookeeper.service.j2
     dest: /lib/systemd/system/zookeeper.service
@@ -107,7 +105,7 @@
 
 - name: Configure Zookeeper
   template:
-    src: zoo.j2
+    src: zoo.cfg.j2
     dest: "{{ zookeeper_install_dir }}/conf/zoo.cfg"
     owner: "{{ zookeeper_user }}"
     group: "{{ zookeeper_group }}"
@@ -116,14 +114,11 @@
 - name: Link /opt/zookeeper to the right version
   file: path=/opt/zookeeper state=link src="{{ zookeeper_install_dir }}"
 
-# - name: Link /etc/zookeeper/conf to /opt/zookeeper/config
-#   file: path=/etc/zookeeper/conf state=link src=/opt/zookeeper/conf
-
 - name: Add Zookeeper's bin dir to the PATH
   copy:
     content: "export PATH=$PATH:/opt/zookeeper/bin"
     dest: "/etc/profile.d/zookeeper_path.sh"
-    mode: 0755
+    mode: u=rwx,g=rx,o=rx
 
 - name: Update the log4j config with saner production values
   template:
@@ -145,4 +140,3 @@
 
 - include_tasks: metrics.yml
   when: exporter.stat.exists
-

core/src/epicli/data/common/ansible/playbooks/roles/zookeeper/templates/zoo.cfg.j2

@@ -0,0 +1,35 @@
+# {{ ansible_managed }}
+
+# ========================= Epiphany built-in settings =========================
+
+# ---  Based on zoo_sample.cfg ---
+
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=5
+# The directory where the snapshot is stored
+dataDir={{ zookeeper_data_dir }}
+# The port at which the clients will connect
+clientPort=2181
+
+# --- Added by Epiphany team ---
+
+# The directory where the transaction logs are stored
+dataLogDir={{ zookeeper_log_dir }}
+
+{% for host in zookeeper_hosts %}
+server.{{loop.index}}={{ host }}:2888:3888
+{% endfor %}
+
+{% if specification.static_config_file.configurable_block is defined
+  and specification.static_config_file.configurable_block is string
+  and specification.static_config_file.configurable_block | length > 0 %}
+# =============== Settings from Epiphany configuration/zookeeper ===============
+
+{{ specification.static_config_file.configurable_block }}
+{% endif %}

core/src/epicli/data/common/ansible/playbooks/roles/zookeeper/templates/zookeeper.service.j2

@@ -10,7 +10,7 @@ Group={{ zookeeper_group }}
 {% if exporter.stat.exists %}
 Environment="SERVER_JVMFLAGS=-javaagent:{{ prometheus_jmx_path }}={{ prometheus_jmx_exporter_web_listen_port }}:{{ prometheus_jmx_config }}"
 {% endif %}
-ExecStart={{zookeeper_install_dir}}/bin/zkServer.sh start-foreground
+ExecStart=/opt/zookeeper/bin/zkServer.sh start-foreground
 Restart=always
 RestartSec=3
 

core/src/epicli/data/common/ansible/playbooks/upgrade.yml

@@ -98,6 +98,16 @@
 #         name: upgrade
 #         tasks_from: filebeat
 
+- hosts: zookeeper
+  serial: 1
+  become: true
+  become_method: sudo
+  tasks:
+    - import_role:
+        name: upgrade
+        tasks_from: zookeeper
+      vars: { lock_file: /var/tmp/zookeeper-upgrade-in-progress.flag }
+
 - hosts: kafka
   serial: 1
   become: true

core/src/epicli/data/common/defaults/configuration/zookeeper.yml

@@ -2,5 +2,17 @@ kind: configuration/zookeeper
 title: "Zookeeper"
 name: default
 specification:
-  version: 3.4.12
-  file_name: "zookeeper-3.4.12.tar.gz"
+  static_config_file:
+    # This block is injected to $ZOOCFGDIR/zoo.cfg
+    configurable_block: |
+      # Limits the number of concurrent connections (at the socket level) that a single client, identified by IP address,
+      # may make to a single member of the ZooKeeper ensemble. This is used to prevent certain classes of DoS attacks,
+      # including file descriptor exhaustion. The default is 60. Setting this to 0 removes the limit.
+      maxClientCnxns=0
+      # --- AdminServer configuration ---
+      # By default the AdminServer is enabled. Disabling it will cause automated test failures.
+      admin.enableServer=true
+      # The address the embedded Jetty server listens on. Defaults to 0.0.0.0.
+      admin.serverAddress=127.0.0.1
+      # The port the embedded Jetty server listens on. Defaults to 8080.
+      admin.serverPort=8008