From ac28454e69e779e8ddd8360a07efe01c360ea605 Mon Sep 17 00:00:00 2001 From: Florian RUYNAT Date: Sun, 12 Apr 2020 19:55:17 +0200 Subject: [PATCH] Update flannel to 0.12.0 (with new manifests) and disable tx/rx offloading in networking test --- README.md | 2 +- docs/flannel.md | 6 ++ roles/download/defaults/main.yml | 2 +- .../network_plugin/flannel/defaults/main.yml | 3 - roles/network_plugin/flannel/tasks/main.yml | 16 ++++ .../flannel/templates/cni-flannel-rbac.yml.j2 | 65 +++++++++++--- .../flannel/templates/cni-flannel.yml.j2 | 88 +++++++++---------- tests/files/packet_centos7-flannel-addons.yml | 3 - tests/testcases/040_check-network-adv.yml | 6 ++ 9 files changed, 129 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 57f4b113917..97cf7859335 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ Note: Upstart/SysV init based OS types are not supported. - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions) - [cilium](https://github.com/cilium/cilium) v1.5.5 - [contiv](https://github.com/contiv/install) v1.2.1 - - [flanneld](https://github.com/coreos/flannel) v0.11.0 + - [flanneld](https://github.com/coreos/flannel) v0.12.0 - [kube-router](https://github.com/cloudnativelabs/kube-router) v0.4.0 - [multus](https://github.com/intel/multus-cni) v3.4.1 - [weave](https://github.com/weaveworks/weave) v2.5.2 diff --git a/docs/flannel.md b/docs/flannel.md index bea14d79c91..d4a9b97139c 100644 --- a/docs/flannel.md +++ b/docs/flannel.md @@ -1,5 +1,11 @@ # Flannel +Flannel is a network fabric for containers, designed for Kubernetes + +**Warning:** You may encounter this [bug](https://github.com/coreos/flannel/pull/1282) with `VXLAN` backend, while waiting on a newer Flannel version the current workaround (`ethtool --offload flannel.1 rx off tx off`) is showcase in kubespray [networking test](tests/testcases/040_check-network-adv.yml:31). + +## Verifying flannel install + * Flannel configuration file should have been created there ```ShellSession diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 5d5cf8a6047..f78fb662efa 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -72,7 +72,7 @@ calico_policy_version: "v3.13.2" calico_typha_version: "v3.13.2" typha_enabled: false -flannel_version: "v0.11.0" +flannel_version: "v0.12.0" flannel_cni_version: "v0.3.0" cni_version: "v0.8.5" diff --git a/roles/network_plugin/flannel/defaults/main.yml b/roles/network_plugin/flannel/defaults/main.yml index f7f773fdc35..09b18fd3b8c 100644 --- a/roles/network_plugin/flannel/defaults/main.yml +++ b/roles/network_plugin/flannel/defaults/main.yml @@ -23,6 +23,3 @@ flannel_memory_limit: 500M flannel_cpu_limit: 300m flannel_memory_requests: 64M flannel_cpu_requests: 150m - -# Legacy directory, will be removed if found. -flannel_cert_dir: /etc/flannel/certs diff --git a/roles/network_plugin/flannel/tasks/main.yml b/roles/network_plugin/flannel/tasks/main.yml index d5a725baff0..d187a8e19cb 100644 --- a/roles/network_plugin/flannel/tasks/main.yml +++ b/roles/network_plugin/flannel/tasks/main.yml @@ -9,3 +9,19 @@ register: flannel_node_manifests when: - inventory_hostname in groups['kube-master'] + +- name: Flannel | Set CNI directory permissions + file: + path: /opt/cni/bin + state: directory + owner: kube + recurse: true + mode: 0755 + register: cni_bin_dir + +- name: Flannel | Copy CNI plugins + unarchive: + src: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz" + dest: "/opt/cni/bin" + mode: 0755 + remote_src: yes diff --git a/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 index 873d0e3f5a8..82e1bc5d8a2 100644 --- a/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 +++ b/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 @@ -3,13 +3,66 @@ apiVersion: v1 kind: ServiceAccount metadata: name: flannel - namespace: "kube-system" + namespace: kube-system +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp.flannel.unprivileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default +{% if podsecuritypolicy_enabled and apparmor_enabled %} + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default +{% endif %} +spec: + privileged: false + volumes: + - configMap + - secret + - emptyDir + - hostPath + allowedHostPaths: + - pathPrefix: "/etc/cni/net.d" + - pathPrefix: "/etc/kube-flannel" + - pathPrefix: "/run/flannel" + readOnlyRootFilesystem: false + # Users and groups + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny + # Privilege Escalation + allowPrivilegeEscalation: false + defaultAllowPrivilegeEscalation: false + # Capabilities + allowedCapabilities: ['NET_ADMIN'] + defaultAddCapabilities: [] + requiredDropCapabilities: [] + # Host namespaces + hostPID: false + hostIPC: false + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + # SELinux + seLinux: + # SELinux is unused in CaaSP + rule: 'RunAsAny' --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: flannel rules: + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['psp.flannel.unprivileged'] - apiGroups: - "" resources: @@ -29,14 +82,6 @@ rules: - nodes/status verbs: - patch - - apiGroups: - - policy - resourceNames: - - privileged - resources: - - podsecuritypolicies - verbs: - - use --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 @@ -49,4 +94,4 @@ roleRef: subjects: - kind: ServiceAccount name: flannel - namespace: "kube-system" \ No newline at end of file + namespace: kube-system diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 index 9afabd15177..135be5c9418 100644 --- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 +++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 @@ -3,28 +3,27 @@ kind: ConfigMap apiVersion: v1 metadata: name: kube-flannel-cfg - namespace: "kube-system" + namespace: kube-system labels: tier: node app: flannel data: cni-conf.json: | { - "name":"cni0", - "cniVersion":"0.3.1", - "plugins":[ + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ { - "type":"flannel", - "delegate":{ - "forceAddress":true, + "type": "flannel", + "delegate": { "hairpinMode": true, - "isDefaultGateway":true + "isDefaultGateway": true } }, { - "type":"portmap", - "capabilities":{ - "portMappings":true + "type": "portmap", + "capabilities": { + "portMappings": true } } ] @@ -41,25 +40,22 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-flannel - namespace: "kube-system" + namespace: kube-system labels: tier: node - k8s-app: flannel + app: flannel spec: selector: matchLabels: - tier: node - k8s-app: flannel + app: flannel template: metadata: labels: tier: node - k8s-app: flannel + app: flannel spec: priorityClassName: system-node-critical serviceAccountName: flannel - nodeSelector: - beta.kubernetes.io/os: linux containers: - name: kube-flannel image: {{ flannel_image_repo }}:{{ flannel_image_tag }} @@ -73,7 +69,9 @@ spec: memory: {{ flannel_memory_requests }} command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"{% if flannel_interface is defined %}, "--iface={{ flannel_interface }}"{% endif %}{% if flannel_interface_regexp is defined %}, "--iface-regex={{ flannel_interface_regexp }}"{% endif %} ] securityContext: - privileged: true + privileged: false + capabilities: + add: ["NET_ADMIN"] env: - name: POD_NAME valueFrom: @@ -83,45 +81,47 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP volumeMounts: - name: run - mountPath: /run - - name: cni - mountPath: /etc/cni/net.d + mountPath: /run/flannel - name: flannel-cfg mountPath: /etc/kube-flannel/ + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/os + operator: In + values: + - linux + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + initContainers: - name: install-cni image: {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} - command: ["/install-cni.sh"] - env: - # The CNI network config to install on each node. - - name: CNI_NETWORK_CONFIG - valueFrom: - configMapKeyRef: - name: kube-flannel-cfg - key: cni-conf.json - - name: CNI_CONF_NAME - value: "10-flannel.conflist" + command: + - cp + args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist volumeMounts: - name: cni - mountPath: /host/etc/cni/net.d - - name: host-cni-bin - mountPath: /host/opt/cni/bin/ + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ hostNetwork: true dnsPolicy: ClusterFirstWithHostNet tolerations: - - operator: Exists - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - - key: CriticalAddonsOnly - operator: "Exists" + - operator: Exists + effect: NoSchedule volumes: - name: run hostPath: - path: /run + path: /run/flannel - name: cni hostPath: path: /etc/cni/net.d diff --git a/tests/files/packet_centos7-flannel-addons.yml b/tests/files/packet_centos7-flannel-addons.yml index 60ccb69c0f9..a90e5eaf5af 100644 --- a/tests/files/packet_centos7-flannel-addons.yml +++ b/tests/files/packet_centos7-flannel-addons.yml @@ -28,6 +28,3 @@ enable_nodelocaldns: false kube_oidc_url: https://accounts.google.com/.well-known/openid-configuration kube_oidc_client_id: kubespray-example - -# Temp set k8s ver to 1.16.8 -kube_version: v1.16.8 \ No newline at end of file diff --git a/tests/testcases/040_check-network-adv.yml b/tests/testcases/040_check-network-adv.yml index de2197e0941..9818ef38ddf 100644 --- a/tests/testcases/040_check-network-adv.yml +++ b/tests/testcases/040_check-network-adv.yml @@ -14,6 +14,12 @@ netchecker_port: 31081 tasks: + - name: Flannel | Disable tx and rx offloading on VXLAN interfaces (see https://github.com/coreos/flannel/pull/1282) + shell: "ethtool --offload flannel.1 rx off tx off" + ignore_errors: true + when: + - kube_network_plugin == 'flannel' + - name: Force binaries directory for Container Linux by CoreOS and Flatcar set_fact: bin_dir: "/opt/bin"