Add identity_allocation_mode support for Cilium (#8430)

Co-authored-by: Emin Aktaş <[email protected]>
Co-authored-by: Yasin Taha Erol <[email protected]>
Signed-off-by: necatican <[email protected]>

Co-authored-by: Emin Aktaş <[email protected]>
Co-authored-by: Yasin Taha Erol <[email protected]>

docs/cilium.md

@@ -15,7 +15,7 @@ balancer deployed by Kubespray and **only contacts the first master**.
 ## Choose Cilium version
 
 ```yml
-cilium_version: v1.9.9
+cilium_version: v1.11.0
 ```
 
 ## Add variable to config
@@ -29,6 +29,16 @@ cilium_config_extra_vars:
   enable-endpoint-routes: true
 ```
 
+## Change Identity Allocation Mode
+
+Cilium assigns an identity for each endpoint. This identity is used to enforce basic connectivity between endpoints.
+
+Cilium currently supports two different identity allocation modes:
+
+- "crd" stores identities in kubernetes as CRDs (custom resource definition).
+  - These can be queried with `kubectl get ciliumid`
+- "kvstore" stores identities in an etcd kvstore.
+
 ## Install Cilium Hubble
 
 k8s-net-cilium.yml:

inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml

@@ -1,3 +1,4 @@
 # see roles/network_plugin/cilium/defaults/main.yml
 
 # cilium_version: "v1.11.0"
+# identity_allocation_mode: kvstore # kvstore or crd

roles/network_plugin/cilium/defaults/main.yml

@@ -6,6 +6,14 @@ cilium_mtu: ""
 cilium_enable_ipv4: true
 cilium_enable_ipv6: false
 
+# Identity allocation mode selects how identities are shared between cilium
+# nodes by setting how they are stored. The options are "crd" or "kvstore".
+# - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+#   These can be queried with:
+#     `kubectl get ciliumid`
+# - "kvstore" stores identities in an etcd kvstore.
+identity_allocation_mode: kvstore
+
 # Etcd SSL dirs
 cilium_cert_dir: /etc/cilium/certs
 kube_etcd_cacert_file: ca.pem

roles/network_plugin/cilium/tasks/check.yml

@@ -6,4 +6,9 @@
     msg: "cilium_ipsec_key should be defined to use cilium_ipsec_enabled"
   when:
     - cilium_ipsec_enabled
-    - cilium_tunnel_mode in ['vxlan']
+    - cilium_tunnel_mode in ['vxlan']
+
+- name: Stop if bad Cilium identity allocation mode
+  assert:
+    that: identity_allocation_mode in ['crd', 'kvstore']
+    msg: "identity_allocation_mode must be either 'crd' or 'kvstore'"

roles/network_plugin/cilium/tasks/install.yml

@@ -13,6 +13,8 @@
     mode: 0750
     owner: root
     group: root
+  when:
+    - identity_allocation_mode == "kvstore"
 
 - name: Cilium | Link etcd certificates for cilium
   file:
@@ -25,6 +27,8 @@
     - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
     - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
     - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
+  when:
+    - identity_allocation_mode == "kvstore"
 
 - name: Cilium | Create hubble dir
   file:

roles/network_plugin/cilium/templates/cilium-config.yml.j2

@@ -5,6 +5,9 @@ metadata:
   name: cilium-config
   namespace: kube-system
 data:
+  identity-allocation-mode: {{ identity_allocation_mode }}
+
+{% if identity_allocation_mode == "kvstore" %}
   # This etcd-config contains the etcd endpoints of your cluster. If you use
   # TLS please make sure you follow the tutorial in https://cilium.link/etcd-config
   etcd-config: |-
@@ -29,6 +32,7 @@ data:
   # https://docs.cilium.io/en/latest/cmdref/kvstore/
   kvstore: etcd
   kvstore-opt: '{"etcd.config": "/var/lib/etcd-config/etcd.config"}'
+{% endif %}
 
   # If you want metrics enabled in all of your Cilium agents, set the port for
   # which the Cilium agents will have their metrics exposed.

roles/network_plugin/cilium/templates/cilium-deploy.yml.j2

@@ -135,12 +135,14 @@ spec:
             periodSeconds: 10
             timeoutSeconds: 3
           volumeMounts:
+{% if identity_allocation_mode == "kvstore" %}
             - mountPath: /var/lib/etcd-config
               name: etcd-config-path
               readOnly: true
             - mountPath: "{{cilium_cert_dir}}"
               name: etcd-secrets
               readOnly: true
+{% endif %}
             - mountPath: /tmp/cilium/config-map
               name: cilium-config-path
               readOnly: true
@@ -153,6 +155,7 @@ spec:
       tolerations:
         - operator: Exists
       volumes:
+{% if identity_allocation_mode == "kvstore" %}
         # To read the etcd config stored in config maps
         - configMap:
             defaultMode: 420
@@ -165,6 +168,7 @@ spec:
         - name: etcd-secrets
           hostPath:
             path: "{{cilium_cert_dir}}"
+{% endif %}
         - configMap:
             name: cilium-config
           name: cilium-config-path

roles/network_plugin/cilium/templates/cilium-ds.yml.j2

@@ -146,12 +146,14 @@ spec:
           mountPath: {{ cri_socket }}
           readOnly: true
 {% endif %}
+{% if identity_allocation_mode == "kvstore" %}
         - mountPath: /var/lib/etcd-config
           name: etcd-config-path
           readOnly: true
         - mountPath: "{{cilium_cert_dir}}"
           name: etcd-secrets
           readOnly: true
+{% endif %}
         - mountPath: /var/lib/cilium/clustermesh
           name: clustermesh-secrets
           readOnly: true
@@ -270,6 +272,7 @@ spec:
           path: /run/xtables.lock
           type: FileOrCreate
         name: xtables-lock
+{% if identity_allocation_mode == "kvstore" %}
         # To read the etcd config stored in config maps
       - configMap:
           defaultMode: 420
@@ -282,6 +285,7 @@ spec:
       - name: etcd-secrets
         hostPath:
           path: "{{cilium_cert_dir}}"
+{% endif %}
         # To read the clustermesh configuration
       - name: clustermesh-secrets
         secret: