Upgrade cilium role (#7521)

* Upgrade cilium roles * Del old test result * Add hubble ui examples * Refactor hubble metrics * Markdown fix pipeline errors * yamllint check and fix * refactor install from #7520 * Docs syntax change (fix) * Cilium set default 1.8.9 * Update cilium version in Readme
kubernetes-sigs · Apr 30, 2021 · 2556eb2 · 2556eb2
1 parent d29ea38
commit 2556eb2
Show file tree

Hide file tree

Showing 21 changed files with 772 additions and 16 deletions.
diff --git a/README.md b/README.md
@@ -137,7 +137,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [cni-plugins](https://github.com/containernetworking/plugins) v0.9.1
   - [calico](https://github.com/projectcalico/calico) v3.17.4
   - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
-  - [cilium](https://github.com/cilium/cilium) v1.8.8
+  - [cilium](https://github.com/cilium/cilium) v1.8.9
   - [flanneld](https://github.com/coreos/flannel) v0.13.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) v1.6.2
   - [kube-router](https://github.com/cloudnativelabs/kube-router) v1.2.2

diff --git a/docs/cilium.md b/docs/cilium.md
@@ -11,3 +11,53 @@ Hence, in this configuration in Kubespray, Cilium will always contact
 the external loadbalancer (even from a node in the control plane)
 and if there is no external load balancer It will ignore any local load
 balancer deployed by Kubespray and **only contacts the first master**.
+
+## Choose Cilium version
+
+```yml
+cilium_version: v1.8.9 ## or v1.9.6
+```
+
+## Add variable to config
+
+Use following variables:
+
+Example:
+
+```yml
+cilium_config_extra_vars:
+  enable-endpoint-routes: true
+```
+
+## Install Cilium Hubble
+
+k8s-net-cilium.yml:
+
+```yml
+cilium_enable_hubble: true ## enable support hubble in cilium
+cilium_hubble_install: true ## install hubble-relay, hubble-ui
+cilium_hubble_tls_generate: true ## install hubble-certgen and generate certificates
+```
+
+To validate that Hubble UI is properly configured, set up a port forwarding for hubble-ui service:
+
+```shell script
+kubectl port-forward -n kube-system svc/hubble-ui 12000:80
+```
+
+and then open [http://localhost:12000/](http://localhost:12000/).
+
+## Hubble metrics
+
+```yml
+cilium_enable_hubble_metrics: true
+cilium_hubble_metrics:
+  - dns
+  - drop
+  - tcp
+  - flow
+  - icmp
+  - http
+```  
+
+[More](https://docs.cilium.io/en/v1.9/operations/metrics/#hubble-exported-metrics)
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
@@ -77,7 +77,7 @@ flannel_version: "v0.13.0"
 cni_version: "v0.9.1"
 weave_version: 2.8.1
 pod_infra_version: "3.3"
-cilium_version: "v1.8.8"
+cilium_version: "v1.8.9"
 kube_ovn_version: "v1.6.2"
 kube_router_version: "v1.2.2"
 multus_version: "v3.7"
@@ -429,6 +429,16 @@ cilium_init_image_repo: "{{ quay_image_repo }}/cilium/cilium-init"
 cilium_init_image_tag: "2019-04-05"
 cilium_operator_image_repo: "{{ quay_image_repo }}/cilium/operator"
 cilium_operator_image_tag: "{{ cilium_version }}"
+cilium_hubble_relay_image_repo: "{{ quay_image_repo }}/cilium/hubble-relay"
+cilium_hubble_relay_image_tag: "{{ cilium_version }}"
+cilium_hubble_certgen_image_repo: "{{ quay_image_repo }}/cilium/certgen"
+cilium_hubble_certgen_image_tag: "v0.1.3"
+cilium_hubble_ui_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui"
+cilium_hubble_ui_image_tag: "v0.7.3"
+cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-backend"
+cilium_hubble_ui_backend_image_tag: "v0.7.3"
+cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy"
+cilium_hubble_envoy_image_tag: "v1.14.5"
 kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn"
 kube_ovn_container_image_tag: "{{ kube_ovn_version }}"
 kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router"

diff --git a/roles/kubernetes-apps/network_plugin/meta/main.yml b/roles/kubernetes-apps/network_plugin/meta/main.yml
@@ -1,10 +1,5 @@
 ---
 dependencies:
-  - role: kubernetes-apps/network_plugin/cilium
-    when: kube_network_plugin == 'cilium' or cilium_deploy_additionally | default(false) | bool
-    tags:
-      - cilium
-
   - role: kubernetes-apps/network_plugin/calico
     when: kube_network_plugin == 'calico'
     tags:

diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml
@@ -22,9 +22,6 @@ cilium_cpu_requests: 100m
 cilium_tunnel_mode: vxlan
 # Optional features
 cilium_enable_prometheus: false
-cilium_enable_hubble_metrics: false
-cilium_enable_hubble: false
-cilium_hubble_metrics: ""
 # Enable if you want to make use of hostPort mappings
 cilium_enable_portmap: false
 # Monitor aggregation level (none/low/medium/maximum)
@@ -55,6 +52,30 @@ cilium_native_routing_cidr: ""
 # IPsec based transparent encryption between nodes
 cilium_ipsec_enabled: false
 
+# Hubble
+### Enable Hubble without install
+cilium_enable_hubble: false
+### Enable Hubble Metrics
+cilium_enable_hubble_metrics: false
+### if cilium_enable_hubble_metrics: true
+cilium_hubble_metrics: {}
+# - dns
+# - drop
+# - tcp
+# - flow
+# - icmp
+# - http
+### Enable Hubble install
+cilium_hubble_install: false
+### Enable auto generate certs if cilium_hubble_install: true
+cilium_hubble_tls_generate: false
+
 # IP address management mode for v1.9+.
 # https://docs.cilium.io/en/v1.9/concepts/networking/ipam/
 cilium_ipam_mode: kubernetes
+
+## A dictionary of extra config variables to add to cilium-config, formatted like:
+##  cilium_config_extra_vars:
+##    var1: "value1"
+##    var2: "value2"
+cilium_config_extra_vars: {}
diff --git a/...apps/network_plugin/cilium/tasks/main.yml → roles/network_plugin/cilium/tasks/apply.yml b/...apps/network_plugin/cilium/tasks/main.yml → roles/network_plugin/cilium/tasks/apply.yml
@@ -7,7 +7,7 @@
     resource: "{{ item.item.type }}"
     filename: "{{ kube_config_dir }}/{{ item.item.file }}"
     state: "latest"
-  with_items: "{{ cilium_node_manifests.results }}"
+  loop: "{{ cilium_node_manifests.results }}"
   when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped
 
 - name: Cilium | Wait for pods to run
@@ -18,3 +18,16 @@
   delay: 10
   ignore_errors: yes
   when: inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Cilium | Hubble install
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "kube-system"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/hubble/{{ item.item.file }}"
+    state: "latest"
+  loop: "{{ cilium_hubble_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0] and not item is skipped
+    - cilium_enable_hubble and cilium_hubble_install
diff --git a/roles/network_plugin/cilium/tasks/install.yml b/roles/network_plugin/cilium/tasks/install.yml
@@ -20,16 +20,27 @@
     dest: "{{ cilium_cert_dir }}/{{ item.d }}"
     state: hard
     force: yes
-  with_items:
+  loop:
     - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
     - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
     - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
 
+- name: Cilium | Create hubble dir
+  file:
+    path: "{{ kube_config_dir }}/addons/hubble"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_hubble_install
+
 - name: Cilium | Create Cilium node manifests
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/{{ item.file }}"
-  with_items:
+  loop:
     - {name: cilium, file: cilium-config.yml, type: cm}
     - {name: cilium, file: cilium-crb.yml, type: clusterrolebinding}
     - {name: cilium, file: cilium-cr.yml, type: clusterrole}
@@ -42,6 +53,25 @@
     - inventory_hostname in groups['kube_control_plane']
     - item.when | default(True) | bool
 
+- name: Cilium | Create Cilium Hubble manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/hubble/{{ item.file }}"
+  loop:
+    - {name: hubble, file: hubble-config.yml, type: cm}
+    - {name: hubble, file: hubble-crb.yml, type: clusterrolebinding}
+    - {name: hubble, file: hubble-cr.yml, type: clusterrole}
+    - {name: hubble, file: hubble-cronjob.yml, type: cronjob, when: "{{ cilium_hubble_tls_generate }}"}
+    - {name: hubble, file: hubble-deploy.yml, type: deploy}
+    - {name: hubble, file: hubble-job.yml, type: job, when: "{{ cilium_hubble_tls_generate }}"}
+    - {name: hubble, file: hubble-sa.yml, type: sa}
+    - {name: hubble, file: hubble-service.yml, type: service}
+  register: cilium_hubble_manifests
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - cilium_enable_hubble and cilium_hubble_install
+    - item.when | default(True) | bool
+
 - name: Cilium | Enable portmap addon
   template:
     src: 000-cilium-portmap.conflist.j2

diff --git a/roles/network_plugin/cilium/tasks/main.yml b/roles/network_plugin/cilium/tasks/main.yml
@@ -2,3 +2,5 @@
 - import_tasks: check.yml
 
 - include_tasks: install.yml
+
+- include_tasks: apply.yml
diff --git a/roles/network_plugin/cilium/tasks/reset.yml b/roles/network_plugin/cilium/tasks/reset.yml
@@ -3,7 +3,7 @@
   include_tasks: reset_iface.yml
   vars:
     iface: "{{ item }}"
-  with_items:
+  loop:
     - cilium_host
     - cilium_net
     - cilium_vxlan
diff --git a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 b/roles/network_plugin/cilium/templates/cilium-config.yml.j2
@@ -154,13 +154,26 @@ data:
   # Hubble settings
 {% if cilium_enable_hubble %}
   enable-hubble: "true"
-  hubble-metrics: "{{ cilium_hubble_metrics }}"
-  hubble-listen-address: ":4244"
 {% if cilium_enable_hubble_metrics %}
   hubble-metrics-server: ":9091"
+  hubble-metrics:
+{% for hubble_metrics_cycle in cilium_hubble_metrics %}
+    {{ hubble_metrics_cycle }}
+{% endfor %}
+{% endif %}
+  hubble-listen-address: ":4244"
+{% if cilium_enable_hubble and cilium_hubble_install %}
+  hubble-disable-tls: "false"
+  hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt
+  hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key
+  hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt
 {% endif %}
 {% endif %}
 
+{% for key, value in cilium_config_extra_vars.items() %}
+  {{ key }}: "{{ value }}"
+{% endfor %}
+
   # IPsec based transparent encryption between nodes
 {% if cilium_ipsec_enabled %}
   enable-ipsec: "true"

diff --git a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2
@@ -58,6 +58,16 @@ rules:
   - ciliumnodes/status
   - ciliumidentities
   - ciliumidentities/status
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints/finalizers
+  - ciliumnodes/finalizers
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumlocalredirectpolicies/finalizers
 {% endif %}
   verbs:
   - '*'
@@ -183,6 +193,16 @@ rules:
   - ciliumnodes/status
   - ciliumidentities
   - ciliumidentities/status
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints/finalizers
+  - ciliumnodes/finalizers
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumlocalredirectpolicies/finalizers
 {% endif %}
   verbs:
   - '*'
diff --git a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2
@@ -24,6 +24,8 @@ subjects:
 - kind: ServiceAccount
   name: cilium
   namespace: kube-system
+{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
 - apiGroup: rbac.authorization.k8s.io
   kind: Group
   name: system:nodes
+{% endif %}
diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
@@ -170,6 +170,11 @@ spec:
         - mountPath: /etc/ipsec
           name: cilium-ipsec-secrets
           readOnly: true
+{% endif %}
+{% if cilium_hubble_install %}
+        - mountPath: /var/lib/cilium/tls/hubble
+          name: hubble-tls
+          readOnly: true
 {% endif %}
       dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
@@ -196,7 +201,11 @@ spec:
               key: wait-bpf-mount
               name: cilium-config
               optional: true
+{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
         image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}"
+{% else %}
+        image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+{% endif %}
         imagePullPolicy: {{ k8s_image_pull_policy }}
         name: clean-cilium-state
         securityContext:
@@ -289,6 +298,25 @@ spec:
       - name: cilium-ipsec-secrets
         secret:
           secretName: cilium-ipsec-keys
+{% endif %}
+{% if cilium_hubble_install %}
+      - name: hubble-tls
+        projected:
+          sources:
+          - secret:
+              name: hubble-server-certs
+              items:
+                - key: tls.crt
+                  path: server.crt
+                - key: tls.key
+                  path: server.key
+              optional: true
+          - configMap:
+              name: hubble-ca-cert
+              items:
+                - key: ca.crt
+                  path: client-ca.crt
+              optional: true
 {% endif %}
   updateStrategy:
     rollingUpdate:
Original file line number	Diff line number	Diff line change
Expand Up		@@ -2,3 +2,5 @@
		- import_tasks: check.yml

		- include_tasks: install.yml

		- include_tasks: apply.yml