From e601ca327095696848ffb3df0e063662ec41dc41 Mon Sep 17 00:00:00 2001
From: Teddy Andrieux <teddy.andrieux@scality.com>
Date: Thu, 5 Dec 2019 16:44:20 +0100
Subject: [PATCH] salt: Store more etcd members informations in pillar

Add all etcd member informations in the pillar retrived from the etcd3
API and use them to compute the `initial-cluster` value for etcd
manifest (filtering out the member with empty name)

Fixes: #2083
---
 salt/_modules/metalk8s_etcd.py              | 20 ++++++-----
 salt/_pillar/metalk8s_etcd.py               |  4 +--
 salt/metalk8s/kubernetes/etcd/installed.sls | 40 +++++++++++++--------
 3 files changed, 40 insertions(+), 24 deletions(-)

diff --git a/salt/_modules/metalk8s_etcd.py b/salt/_modules/metalk8s_etcd.py
index 775e6128e6..5b2837f1b0 100644
--- a/salt/_modules/metalk8s_etcd.py
+++ b/salt/_modules/metalk8s_etcd.py
@@ -176,14 +176,14 @@ def check_etcd_health(
     else:
         return "cluster is healthy"
 
+
 def get_etcd_member_list(
         endpoint=None,
         nodes=None,
         ca_cert='/etc/kubernetes/pki/etcd/ca.crt',
         cert_key='/etc/kubernetes/pki/etcd/salt-master-etcd-client.key',
         cert_cert='/etc/kubernetes/pki/etcd/salt-master-etcd-client.crt'):
-    '''Get the list of etcd members peer urls using the python etcd3 client.'''
-    member_peer_urls = []
+    '''Get the list of etcd members using the python etcd3 client.'''
     if not endpoint:
         # If we have no endpoint get it from mine
         try:
@@ -194,14 +194,18 @@ def get_etcd_member_list(
                 cert_cert=cert_cert
             )
         except:
-            return member_peer_urls
+            return []
+
     with etcd3.client(host=endpoint,
                       ca_cert=ca_cert,
                       cert_key=cert_key,
                       cert_cert=cert_cert,
                       timeout=TIMEOUT) as etcd:
-        for member in etcd.members:
-            member_peer_urls.append('{0}={1}'.format(
-                member.name,
-                member.peer_urls[0]))
-    return list(set(member_peer_urls))
\ No newline at end of file
+        return [
+            {
+                'id': member.id,
+                'name': member.name,
+                'peer_urls': list(member.peer_urls),
+                'client_urls': list(member.client_urls)
+            } for member in etcd.members
+        ]
diff --git a/salt/_pillar/metalk8s_etcd.py b/salt/_pillar/metalk8s_etcd.py
index 54b6e73197..a0265b25ac 100644
--- a/salt/_pillar/metalk8s_etcd.py
+++ b/salt/_pillar/metalk8s_etcd.py
@@ -13,8 +13,8 @@ def _load_members(pillar):
     errors = []
     try:
         members = __salt__['metalk8s_etcd.get_etcd_member_list'](
-                    nodes = pillar['metalk8s']['nodes']
-                  )
+            nodes=pillar['metalk8s']['nodes']
+        )
     except Exception as exc:
         members = []
         errors.append(
diff --git a/salt/metalk8s/kubernetes/etcd/installed.sls b/salt/metalk8s/kubernetes/etcd/installed.sls
index 4da4d0dd7a..d233fa11bc 100644
--- a/salt/metalk8s/kubernetes/etcd/installed.sls
+++ b/salt/metalk8s/kubernetes/etcd/installed.sls
@@ -4,19 +4,31 @@ include:
   - metalk8s.kubernetes.ca.etcd.advertised
   - .certs
 
-{%- set host_name = grains['id'] %}
-{%- set host = grains['metalk8s']['control_plane_ip'] %}
+{%- set node_name = grains['id'] %}
+{%- set node_ip = grains['metalk8s']['control_plane_ip'] %}
 
-{%- set endpoint  = host_name ~ '=https://' ~ host ~ ':2380' %}
+{%- set endpoint = 'https://' ~ node_ip ~ ':2380' %}
 
-{#- Get the list of existing etcd node. #}
-{%- set etcd_endpoints = pillar.metalk8s.etcd.members %}
+{#- Get the list of existing etcd member. #}
+{%- set etcd_members = pillar.metalk8s.etcd.members %}
 
 {#- Compute the initial state according to the existing list of node. #}
-{%- set state = "existing" if etcd_endpoints else "new" %}
+{%- set state = "existing" if etcd_members else "new" %}
 
-{#- Add ourselves to the list. #}
-{%- do etcd_endpoints.append(endpoint) %}
+{%- set etcd_endpoints = {} %}
+{#- NOTE: Filter out members with empty name as they are not started yet. #}
+{%- for member in etcd_members | selectattr('name') %}
+  {#- NOTE: Only take first peer_urls for endpoint. #}
+  {%- do etcd_endpoints.update({member['name']: member['peer_urls'][0]}) %}
+{%- endfor %}
+
+{#- Add ourselves to the endpoints. #}
+{%- do etcd_endpoints.update({node_name: endpoint}) %}
+
+{%- set etcd_initial_cluster = [] %}
+{%- for name, ep in etcd_endpoints.items() %}
+  {%- do etcd_initial_cluster.append(name ~ '=' ~ ep) %}
+{%- endfor %}
 
 Create etcd database directory:
   file.directory:
@@ -41,17 +53,17 @@ Create local etcd Pod manifest:
         image_name: {{ build_image_name('etcd') }}
         command:
           - etcd
-          - --advertise-client-urls=https://{{ host }}:2379
+          - --advertise-client-urls=https://{{ node_ip }}:2379
           - --cert-file=/etc/kubernetes/pki/etcd/server.crt
           - --client-cert-auth=true
           - --data-dir=/var/lib/etcd
-          - --initial-advertise-peer-urls=https://{{ host }}:2380
-          - --initial-cluster={{ etcd_endpoints|unique|join(',') }}
+          - --initial-advertise-peer-urls=https://{{ node_ip }}:2380
+          - --initial-cluster={{ etcd_initial_cluster| sort | join(',') }}
           - --initial-cluster-state={{ state }}
           - --key-file=/etc/kubernetes/pki/etcd/server.key
-          - --listen-client-urls=https://127.0.0.1:2379,https://{{ host }}:2379
-          - --listen-peer-urls=https://{{ host }}:2380
-          - --name={{ host_name }}
+          - --listen-client-urls=https://127.0.0.1:2379,https://{{ node_ip }}:2379
+          - --listen-peer-urls=https://{{ node_ip }}:2380
+          - --name={{ node_name }}
           - --peer-cert-file=/etc/kubernetes/pki/etcd/peer.crt
           - --peer-client-cert-auth=true
           - --peer-key-file=/etc/kubernetes/pki/etcd/peer.key