From 3b25c0c66a40bb17bc70cddc8c7ebb2790fbfd30 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Wed, 11 Dec 2019 17:47:03 +0200 Subject: [PATCH 1/6] Add Calico v3.10.2 --- .../k8s-1.15.yaml.template | 18 +- .../k8s-1.16.yaml.template | 1091 +++++++++++++++++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 21 +- 3 files changed, 1118 insertions(+), 12 deletions(-) create mode 100644 upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template index 8a9a52664cbdb..f6472fdb23dba 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template @@ -1,10 +1,10 @@ -# Canal Version v3.10.1 +# Canal Version v3.10.2 # https://docs.projectcalico.org/v3.10/release-notes/#v3101 # This manifest includes the following component versions: -# calico/cni:v3.10.1 -# calico/node:v3.10.1 -# calico/pod2daemon-flexvol:v3.10.1 -# calico/typha:v3.10.1 +# calico/cni:v3.10.2 +# calico/node:v3.10.2 +# calico/pod2daemon-flexvol:v3.10.2 +# calico/typha:v3.10.2 # quay.io/coreos/flannel:v0.11.0 # Source: calico/templates/calico-config.yaml @@ -520,7 +520,7 @@ spec: securityContext: fsGroup: 65534 containers: - - image: calico/typha:v3.10.1 + - image: calico/typha:v3.10.2 name: calico-typha ports: - containerPort: 5473 @@ -634,7 +634,7 @@ spec: # This container installs the Calico CNI binaries # and CNI network config file on each node. - name: install-cni - image: calico/cni:v3.10.1 + image: calico/cni:v3.10.2 command: ["/install-cni.sh"] env: # Name of the CNI config file to create. @@ -668,7 +668,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: calico/pod2daemon-flexvol:v3.10.1 + image: calico/pod2daemon-flexvol:v3.10.2 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -677,7 +677,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: calico/node:v3.10.1 + image: calico/node:v3.10.2 env: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template new file mode 100644 index 0000000000000..e92a8e9920b06 --- /dev/null +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template @@ -0,0 +1,1091 @@ +--- +# Pulled and modified from: https://docs.projectcalico.org/v3.9/manifests/calico-typha.yaml + +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" +data: + # You must set a non-zero value for Typha replicas below. + typha_service_name: "{{- if .Networking.Calico.TyphaReplicas -}}calico-typha{{- else -}}none{{- end -}}" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use + {{- if .Networking.Calico.MTU }} + veth_mtu: "{{ .Networking.Calico.MTU }}" + {{- else }} + veth_mtu: "{{- if eq .CloudProvider "openstack" -}}1430{{- else -}}1440{{- end -}}" + {{- end }} + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + } + ] + } + +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: FelixConfiguration + plural: felixconfigurations + singular: felixconfiguration +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMBlock + plural: ipamblocks + singular: ipamblock + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BlockAffinity + plural: blockaffinities + singular: blockaffinity + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMHandle + plural: ipamhandles + singular: ipamhandle + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPAMConfig + plural: ipamconfigs + singular: ipamconfig + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPPeer + plural: bgppeers + singular: bgppeer + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: BGPConfiguration + plural: bgpconfigurations + singular: bgpconfiguration + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: IPPool + plural: ippools + singular: ippool + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: HostEndpoint + plural: hostendpoints + singular: hostendpoint + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: ClusterInformation + plural: clusterinformations + singular: clusterinformation + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkPolicy + plural: globalnetworkpolicies + singular: globalnetworkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Cluster + group: crd.projectcalico.org + version: v1 + names: + kind: GlobalNetworkSet + plural: globalnetworksets + singular: globalnetworkset + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkPolicy + plural: networkpolicies + singular: networkpolicy + +--- + +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" +spec: + scope: Namespaced + group: crd.projectcalico.org + version: v1 + names: + kind: NetworkSet + plural: networksets + singular: networkset +--- +# Source: calico/templates/rbac.yaml + +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers + labels: + role.kubernetes.io/networking: "1" +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - create + - update +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers + labels: + role.kubernetes.io/networking: "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node + labels: + role.kubernetes.io/networking: "1" +rules: + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only requried for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node + labels: + role.kubernetes.io/networking: "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system + +{{ if .Networking.Calico.TyphaReplicas -}} +--- +# Source: calico/templates/calico-typha.yaml +# This manifest creates a Service, which will be backed by Calico's Typha daemon. +# Typha sits in between Felix and the API server, reducing Calico's load on the API server. + +apiVersion: v1 +kind: Service +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha + role.kubernetes.io/networking: "1" +spec: + ports: + - port: 5473 + protocol: TCP + targetPort: calico-typha + name: calico-typha + selector: + k8s-app: calico-typha + +--- + +# This manifest creates a Deployment of Typha to back the above service. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha + role.kubernetes.io/networking: "1" +spec: + # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the + # typha_service_name variable in the calico-config ConfigMap above. + # + # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential + # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In + # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade. + replicas: {{ or .Networking.Calico.TyphaReplicas "0" }} + revisionHistoryLimit: 2 + selector: + matchLabels: + k8s-app: calico-typha + template: + metadata: + labels: + k8s-app: calico-typha + role.kubernetes.io/networking: "1" + annotations: + # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical + # add-on, ensuring it gets priority scheduling and that its resources are reserved + # if it ever gets evicted. + scheduler.alpha.kubernetes.io/critical-pod: '' + cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + tolerations: + - key: CriticalAddonsOnly + operator: Exists + # Since Calico can't network a pod until Typha is up, we need to run Typha itself + # as a host-networked pod. + serviceAccountName: calico-node + priorityClassName: system-cluster-critical + # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 + securityContext: + fsGroup: 65534 + containers: + - image: calico/typha:v3.10.2 + name: calico-typha + ports: + - containerPort: 5473 + name: calico-typha + protocol: TCP + env: + # Enable "info" logging by default. Can be set to "debug" to increase verbosity. + - name: TYPHA_LOGSEVERITYSCREEN + value: "info" + # Disable logging to file and syslog since those don't make sense in Kubernetes. + - name: TYPHA_LOGFILEPATH + value: "none" + - name: TYPHA_LOGSEVERITYSYS + value: "none" + # Monitor the Kubernetes API to find the number of running instances and rebalance + # connections. + - name: TYPHA_CONNECTIONREBALANCINGMODE + value: "kubernetes" + - name: TYPHA_DATASTORETYPE + value: "kubernetes" + - name: TYPHA_HEALTHENABLED + value: "true" + - name: TYPHA_PROMETHEUSMETRICSENABLED + value: "{{- or .Networking.Calico.TyphaPrometheusMetricsEnabled "false" }}" + - name: TYPHA_PROMETHEUSMETRICSPORT + value: "{{- or .Networking.Calico.TyphaPrometheusMetricsPort "9093" }}" + livenessProbe: + httpGet: + path: /liveness + port: 9098 + host: localhost + periodSeconds: 30 + initialDelaySeconds: 30 + securityContext: + runAsNonRoot: true + allowPrivilegeEscalation: false + readinessProbe: + httpGet: + path: /readiness + port: 9098 + host: localhost + periodSeconds: 10 + +--- + +# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: calico-typha + namespace: kube-system + labels: + k8s-app: calico-typha + role.kubernetes.io/networking: "1" +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-typha +{{- end -}} +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node + role.kubernetes.io/networking: "1" +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + role.kubernetes.io/networking: "1" + annotations: + # This, along with the CriticalAddonsOnly toleration below, + # marks the pod as a critical add-on, ensuring it gets + # priority scheduling and that its resources are reserved + # if it ever gets evicted. + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: calico/cni:v3.10.2 + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: calico/cni:v3.10.2 + command: ["/install-cni.sh"] + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver + image: calico/pod2daemon-flexvol:v3.10.2 + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: calico/node:v3.10.2 + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Typha support: controlled by the ConfigMap. + - name: FELIX_TYPHAK8SSERVICENAME + valueFrom: + configMapKeyRef: + name: calico-config + key: typha_service_name + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + # was value: "k8s,bgp" + value: "kops,bgp" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "{{ .KubeControllerManager.ClusterCIDR }}" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to the desired level + - name: FELIX_LOGSEVERITYSCREEN + value: "{{- or .Networking.Calico.LogSeverityScreen "info" }}" + - name: FELIX_HEALTHENABLED + value: "true" + + # kops additions + # Set Felix iptables binary variant, Legacy or NFT + - name: FELIX_IPTABLESBACKEND + value: "{{- or .Networking.Calico.IptablesBackend "Legacy" }}" + # Set to enable the experimental Prometheus metrics server + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "{{- or .Networking.Calico.PrometheusMetricsEnabled "false" }}" + # TCP port that the Prometheus metrics server should bind to + - name: FELIX_PROMETHEUSMETRICSPORT + value: "{{- or .Networking.Calico.PrometheusMetricsPort "9091" }}" + # Enable Prometheus Go runtime metrics collection + - name: FELIX_PROMETHEUSGOMETRICSENABLED + value: "{{- or .Networking.Calico.PrometheusGoMetricsEnabled "true" }}" + # Enable Prometheus process metrics collection + - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED + value: "{{- or .Networking.Calico.PrometheusProcessMetricsEnabled "true" }}" + securityContext: + privileged: true + resources: + requests: + cpu: 90m + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent + # Used to install Flex Volume Driver + - name: flexvol-driver-host + hostPath: + type: DirectoryOrCreate + path: "{{- or .Kubelet.VolumePluginDirectory "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" }}nodeagent~uds" +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" + +--- +# Source: calico/templates/calico-kube-controllers.yaml + +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + role.kubernetes.io/networking: "1" +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + role.kubernetes.io/networking: "1" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + nodeSelector: + beta.kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: calico/kube-controllers:v3.10.2 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" + +{{ if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}} +# This manifest installs the k8s-ec2-srcdst container, which disables +# src/dst ip checks to allow BGP to function for calico for hosts within subnets +# This only applies for AWS environments. +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: k8s-ec2-srcdst + labels: + role.kubernetes.io/networking: "1" +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: k8s-ec2-srcdst + namespace: kube-system + labels: + role.kubernetes.io/networking: "1" +--- + +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: k8s-ec2-srcdst + labels: + role.kubernetes.io/networking: "1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: k8s-ec2-srcdst +subjects: +- kind: ServiceAccount + name: k8s-ec2-srcdst + namespace: kube-system + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k8s-ec2-srcdst + namespace: kube-system + labels: + k8s-app: k8s-ec2-srcdst + role.kubernetes.io/networking: "1" +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: k8s-ec2-srcdst + template: + metadata: + labels: + k8s-app: k8s-ec2-srcdst + role.kubernetes.io/networking: "1" + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + serviceAccountName: k8s-ec2-srcdst + priorityClassName: system-cluster-critical + containers: + - image: ottoyiu/k8s-ec2-srcdst:v0.2.2 + name: k8s-ec2-srcdst + resources: + requests: + cpu: 10m + memory: 64Mi + env: + - name: AWS_REGION + value: {{ Region }} + volumeMounts: + - name: ssl-certs + mountPath: "/etc/ssl/certs/ca-certificates.crt" + readOnly: true + imagePullPolicy: "Always" + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" + nodeSelector: + node-role.kubernetes.io/master: "" +{{- end -}} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 64fde695cf691..ced28e13b8ac4 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -829,6 +829,7 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { "k8s-1.7": "2.6.12-kops.1", "k8s-1.7-v3": "3.8.0-kops.2", "k8s-1.12": "3.9.3-kops.2", + "k8s-1.16": "3.10.2-kops.1", } { @@ -840,7 +841,21 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { Version: fi.String(versions[id]), Selector: networkingSelector, Manifest: fi.String(location), - KubernetesVersion: ">=1.12.0", + KubernetesVersion: ">=1.12.0 <1.16.0", + Id: id, + }) + } + + { + id := "k8s-1.16" + location := key + "/" + id + ".yaml" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(versions[id]), + Selector: networkingSelector, + Manifest: fi.String(location), + KubernetesVersion: ">=1.16.0", Id: id, }) } @@ -911,8 +926,8 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { "k8s-1.6": "2.4.2-kops.2", "k8s-1.8": "2.6.7-kops.3", "k8s-1.9": "3.2.3-kops.1", - "k8s-1.12": "3.7.4", - "k8s-1.15": "3.10.1-kops.2", + "k8s-1.12": "3.7.4-kops.1", + "k8s-1.15": "3.10.2-kops.1", } { id := "pre-k8s-1.6" From 346d0ba9bc6596b3418ef2cc9f7e1aa2c82f1765 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Thu, 12 Dec 2019 15:39:34 +0200 Subject: [PATCH 2/6] Make templates easier to compare with official manifests --- .../k8s-1.15.yaml.template | 75 ++++++++----------- .../k8s-1.16.yaml.template | 23 +++--- 2 files changed, 44 insertions(+), 54 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template index f6472fdb23dba..6281b58f8dc16 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template @@ -1,12 +1,6 @@ -# Canal Version v3.10.2 -# https://docs.projectcalico.org/v3.10/release-notes/#v3101 -# This manifest includes the following component versions: -# calico/cni:v3.10.2 -# calico/node:v3.10.2 -# calico/pod2daemon-flexvol:v3.10.2 -# calico/typha:v3.10.2 -# quay.io/coreos/flannel:v0.11.0 +{{- /* Pulled and modified from: https://docs.projectcalico.org/v3.10/manifests/canal.yaml */ -}} +--- # Source: calico/templates/calico-config.yaml # This ConfigMap is used to configure a self-hosted Canal installation. kind: ConfigMap @@ -15,6 +9,7 @@ metadata: name: canal-config namespace: kube-system data: + # Typha is disabled. typha_service_name: "{{ if .Networking.Canal.TyphaReplicas }}calico-typha{{ else }}none{{ end }}" # The interface used by canal for host <-> host communication. # If left blank, then the interface is chosen using the node's @@ -70,7 +65,6 @@ data: } --- - # Source: calico/templates/kdd-crds.yaml apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition @@ -278,15 +272,15 @@ spec: kind: NetworkSet plural: networksets singular: networkset - --- +# Source: calico/templates/rbac.yaml # Include a clusterrole for the calico-node DaemonSet, # and bind it to the calico-node serviceaccount. kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: calico + name: calico-node rules: # The CNI plugin needs to get pods, nodes, and namespaces. - apiGroups: [""] @@ -422,7 +416,6 @@ subjects: name: canal namespace: kube-system --- -# Bind the Calico ClusterRole to the canal ServiceAccount. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -430,20 +423,12 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: calico + name: calico-node subjects: - kind: ServiceAccount name: canal namespace: kube-system ---- - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: canal - namespace: kube-system - --- {{- if .Networking.Canal.TyphaReplicas }} @@ -500,14 +485,11 @@ spec: scheduler.alpha.kubernetes.io/critical-pod: '' cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' spec: - tolerations: - - key: CriticalAddonsOnly - operator: Exists + nodeSelector: + kubernetes.io/role: master # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. hostNetwork: true - nodeSelector: - kubernetes.io/role: master tolerations: # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly @@ -583,9 +565,9 @@ spec: {{- end }} --- - -# This manifest installs the calico/node container, as well -# as the Calico CNI plugins and network config on +# Source: calico/templates/calico-node.yaml +# This manifest installs the canal container, as well +# as the CNI plugins and network config on # each master and worker node in a Kubernetes cluster. kind: DaemonSet apiVersion: apps/v1 @@ -613,7 +595,6 @@ spec: # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' spec: - priorityClassName: system-node-critical nodeSelector: beta.kubernetes.io/os: linux hostNetwork: true @@ -630,8 +611,9 @@ spec: # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical initContainers: - # This container installs the Calico CNI binaries + # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni image: calico/cni:v3.10.2 @@ -673,7 +655,7 @@ spec: - name: flexvol-driver-host mountPath: /host/driver containers: - # Runs calico/node container on each Kubernetes node. This + # Runs canal container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node @@ -716,6 +698,17 @@ spec: # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "{{- or .Networking.Canal.DefaultEndpointToHostAction "ACCEPT" }}" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + # Set Felix logging to "INFO" + - name: FELIX_LOGSEVERITYSCREEN + value: "{{- or .Networking.Canal.LogSeveritySys "INFO" }}" + - name: FELIX_HEALTHENABLED + value: "true" - name: FELIX_IPINIPMTU valueFrom: configMapKeyRef: @@ -724,15 +717,6 @@ spec: # Set Felix iptables binary variant, Legacy or NFT - name: FELIX_IPTABLESBACKEND value: "{{- or .Networking.Canal.IptablesBackend "Legacy" }}" - # Disable IPv6 on Kubernetes. - - name: FELIX_IPV6SUPPORT - value: "false" - # Set Felix logging to "INFO" - - name: FELIX_LOGSEVERITYSCREEN - value: "{{- or .Networking.Canal.LogSeveritySys "INFO" }}" - # Set Felix endpoint to host default action to ACCEPT. - - name: FELIX_DEFAULTENDPOINTTOHOSTACTION - value: "{{- or .Networking.Canal.DefaultEndpointToHostAction "ACCEPT" }}" # Controls whether Felix inserts rules to the top of iptables chains, or appends to the bottom - name: FELIX_CHAININSERTMODE value: "{{- or .Networking.Canal.ChainInsertMode "insert" }}" @@ -748,8 +732,6 @@ spec: # Enable Prometheus process metrics collection - name: FELIX_PROMETHEUSPROCESSMETRICSENABLED value: "{{- or .Networking.Canal.PrometheusProcessMetricsEnabled "true" }}" - - name: FELIX_HEALTHENABLED - value: "true" securityContext: privileged: true resources: @@ -856,3 +838,10 @@ spec: hostPath: type: DirectoryOrCreate path: "{{- or .Kubelet.VolumePluginDirectory "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/" }}nodeagent~uds" +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: canal + namespace: kube-system diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template index e92a8e9920b06..0ccea9ef28060 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template @@ -1,6 +1,6 @@ ---- -# Pulled and modified from: https://docs.projectcalico.org/v3.9/manifests/calico-typha.yaml +{{- /* Pulled and modified from: https://docs.projectcalico.org/v3.10/manifests/calico-typha.yaml */ -}} +--- # Source: calico/templates/calico-config.yaml # This ConfigMap is used to configure a self-hosted Calico installation. kind: ConfigMap @@ -570,8 +570,9 @@ spec: beta.kubernetes.io/os: linux hostNetwork: true tolerations: - - key: CriticalAddonsOnly - operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node @@ -723,12 +724,6 @@ spec: # Name of the CNI config file to create. - name: CNI_CONF_NAME value: "10-calico.conflist" - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: calico-config - key: veth_mtu # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: @@ -740,6 +735,12 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu # Prevents the container from sleeping forever. - name: SLEEP value: "false" @@ -815,7 +816,7 @@ spec: # Disable IPv6 on Kubernetes. - name: FELIX_IPV6SUPPORT value: "false" - # Set Felix logging to the desired level + # Set Felix logging to "info" - name: FELIX_LOGSEVERITYSCREEN value: "{{- or .Networking.Calico.LogSeverityScreen "info" }}" - name: FELIX_HEALTHENABLED From f6193e0c41bb1a63dd5a00e4de27c07ad93a6f52 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Thu, 12 Dec 2019 15:56:54 +0200 Subject: [PATCH 3/6] Fix indent of metadata.name field for felixconfigurations.crd.projectcalico.org --- .../networking.projectcalico.org.canal/k8s-1.15.yaml.template | 2 +- .../addons/networking.projectcalico.org/k8s-1.16.yaml.template | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template index 6281b58f8dc16..7e62c32ee955f 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template @@ -69,7 +69,7 @@ data: apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: felixconfigurations.crd.projectcalico.org + name: felixconfigurations.crd.projectcalico.org spec: scope: Cluster group: crd.projectcalico.org diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template index 0ccea9ef28060..5c5323a393479 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template @@ -59,7 +59,7 @@ data: apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: felixconfigurations.crd.projectcalico.org + name: felixconfigurations.crd.projectcalico.org labels: role.kubernetes.io/networking: "1" spec: From 7ef9d0a5c183be84673b781b509fe82cfd4b3e62 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Thu, 12 Dec 2019 16:12:28 +0200 Subject: [PATCH 4/6] Add role.kubernetes.io/networking labels for Canal --- .../k8s-1.15.yaml.template | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template index 7e62c32ee955f..8ac9499c1880b 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template @@ -8,6 +8,8 @@ apiVersion: v1 metadata: name: canal-config namespace: kube-system + labels: + role.kubernetes.io/networking: "1" data: # Typha is disabled. typha_service_name: "{{ if .Networking.Canal.TyphaReplicas }}calico-typha{{ else }}none{{ end }}" @@ -70,6 +72,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: felixconfigurations.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -84,6 +88,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: ipamblocks.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -99,6 +105,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: blockaffinities.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -114,6 +122,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: ipamhandles.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -129,6 +139,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: ipamconfigs.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -144,6 +156,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: bgppeers.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -159,6 +173,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: bgpconfigurations.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -174,6 +190,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: ippools.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -189,6 +207,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: hostendpoints.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -204,6 +224,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clusterinformations.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -219,6 +241,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: globalnetworkpolicies.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -234,6 +258,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: globalnetworksets.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Cluster group: crd.projectcalico.org @@ -249,6 +275,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: networkpolicies.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Namespaced group: crd.projectcalico.org @@ -264,6 +292,8 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: networksets.crd.projectcalico.org + labels: + role.kubernetes.io/networking: "1" spec: scope: Namespaced group: crd.projectcalico.org @@ -281,6 +311,8 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: calico-node + labels: + role.kubernetes.io/networking: "1" rules: # The CNI plugin needs to get pods, nodes, and namespaces. - apiGroups: [""] @@ -384,6 +416,8 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: flannel + labels: + role.kubernetes.io/networking: "1" rules: - apiGroups: [""] resources: @@ -407,6 +441,8 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: canal-flannel + labels: + role.kubernetes.io/networking: "1" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -420,6 +456,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: canal-calico + labels: + role.kubernetes.io/networking: "1" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -442,6 +480,7 @@ metadata: namespace: kube-system labels: k8s-app: calico-typha + role.kubernetes.io/networking: "1" spec: ports: - port: 5473 @@ -462,6 +501,7 @@ metadata: namespace: kube-system labels: k8s-app: calico-typha + role.kubernetes.io/networking: "1" spec: # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the # typha_service_name variable in the canal-config ConfigMap above. @@ -478,6 +518,7 @@ spec: metadata: labels: k8s-app: calico-typha + role.kubernetes.io/networking: "1" annotations: # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical # add-on, ensuring it gets priority scheduling and that its resources are reserved @@ -557,6 +598,7 @@ metadata: namespace: kube-system labels: k8s-app: calico-typha + role.kubernetes.io/networking: "1" spec: maxUnavailable: 1 selector: @@ -576,6 +618,7 @@ metadata: namespace: kube-system labels: k8s-app: canal + role.kubernetes.io/networking: "1" spec: selector: matchLabels: @@ -588,6 +631,7 @@ spec: metadata: labels: k8s-app: canal + role.kubernetes.io/networking: "1" annotations: # This, along with the CriticalAddonsOnly toleration below, # marks the pod as a critical add-on, ensuring it gets @@ -845,3 +889,5 @@ kind: ServiceAccount metadata: name: canal namespace: kube-system + labels: + role.kubernetes.io/networking: "1" From a806f10b4ed551fb960ee6bdb4ea8bf2c7029c8b Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Thu, 12 Dec 2019 17:47:09 +0200 Subject: [PATCH 5/6] Make Calico-Typha and Canal templates easier to compare to each other --- .../k8s-1.15.yaml.template | 59 +++++++++++-------- .../k8s-1.16.yaml.template | 10 +++- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template index 8ac9499c1880b..e2d3830c842fa 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template @@ -22,8 +22,12 @@ data: # the pod network. masquerade: "true" - # MTU default is 1500, can be overridden - veth_mtu: "{{- or .Networking.Canal.MTU "1500" }}" + # Configure the MTU to use + {{- if .Networking.Canal.MTU }} + veth_mtu: "{{ .Networking.Canal.MTU }}" + {{- else }} + veth_mtu: "{{- if eq .CloudProvider "openstack" -}}1430{{- else -}}1440{{- end -}}" + {{- end }} # The CNI network configuration to install on each node. The special # values in this config will be automatically populated. @@ -36,8 +40,8 @@ data: "type": "calico", "log_level": "info", "datastore_type": "kubernetes", - "mtu": __CNI_MTU__, "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, "ipam": { "type": "host-local", "subnet": "usePodCidr" @@ -467,9 +471,9 @@ subjects: name: canal namespace: kube-system +{{ if .Networking.Canal.TyphaReplicas -}} --- -{{- if .Networking.Canal.TyphaReplicas }} - +# Source: calico/templates/calico-typha.yaml # This manifest creates a Service, which will be backed by Calico's Typha daemon. # Typha sits in between Felix and the API server, reducing Calico's load on the API server. @@ -527,9 +531,8 @@ spec: cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' spec: nodeSelector: + beta.kubernetes.io/os: linux kubernetes.io/role: master - # Since Calico can't network a pod until Typha is up, we need to run Typha itself - # as a host-networked pod. hostNetwork: true tolerations: # Mark the pod as a critical add-on for rescheduling. @@ -537,6 +540,8 @@ spec: operator: Exists - key: "node-role.kubernetes.io/master" effect: NoSchedule + # Since Calico can't network a pod until Typha is up, we need to run Typha itself + # as a host-networked pod. serviceAccountName: canal priorityClassName: system-cluster-critical # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 @@ -604,8 +609,8 @@ spec: selector: matchLabels: k8s-app: calico-typha - {{- end }} + --- # Source: calico/templates/calico-node.yaml # This manifest installs the canal container, as well @@ -666,12 +671,6 @@ spec: # Name of the CNI config file to create. - name: CNI_CONF_NAME value: "10-canal.conflist" - # CNI MTU Config variable - - name: CNI_MTU - valueFrom: - configMapKeyRef: - name: canal-config - key: veth_mtu # The CNI network config to install on each node. - name: CNI_NETWORK_CONFIG valueFrom: @@ -683,6 +682,12 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: canal-config + key: veth_mtu # Prevents the container from sleeping forever. - name: SLEEP value: "false" @@ -732,13 +737,17 @@ spec: value: "none" # Cluster type to identify the deployment type - name: CLUSTER_TYPE + # was value: "k8s,bgp" value: "k8s,canal" - # Period, in seconds, at which felix re-applies all iptables state - - name: FELIX_IPTABLESREFRESHINTERVAL - value: "60" # No IP address needed. - name: IP value: "" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: canal-config + key: veth_mtu # Disable file logging so `kubectl logs` works. - name: CALICO_DISABLE_FILE_LOGGING value: "true" @@ -753,17 +762,17 @@ spec: value: "{{- or .Networking.Canal.LogSeveritySys "INFO" }}" - name: FELIX_HEALTHENABLED value: "true" - - name: FELIX_IPINIPMTU - valueFrom: - configMapKeyRef: - name: canal-config - key: veth_mtu - # Set Felix iptables binary variant, Legacy or NFT - - name: FELIX_IPTABLESBACKEND - value: "{{- or .Networking.Canal.IptablesBackend "Legacy" }}" + + # kops additions # Controls whether Felix inserts rules to the top of iptables chains, or appends to the bottom - name: FELIX_CHAININSERTMODE value: "{{- or .Networking.Canal.ChainInsertMode "insert" }}" + # Set Felix iptables binary variant, Legacy or NFT + - name: FELIX_IPTABLESBACKEND + value: "{{- or .Networking.Canal.IptablesBackend "Legacy" }}" + # Period, in seconds, at which felix re-applies all iptables state + - name: FELIX_IPTABLESREFRESHINTERVAL + value: "60" # Set to enable the experimental Prometheus metrics server - name: FELIX_PROMETHEUSMETRICSENABLED value: "{{- or .Networking.Canal.PrometheusMetricsEnabled "false" }}" diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template index 5c5323a393479..7bb17dc4064e3 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template @@ -568,11 +568,14 @@ spec: spec: nodeSelector: beta.kubernetes.io/os: linux + kubernetes.io/role: master hostNetwork: true tolerations: # Mark the pod as a critical add-on for rescheduling. - key: CriticalAddonsOnly operator: Exists + - key: "node-role.kubernetes.io/master" + effect: NoSchedule # Since Calico can't network a pod until Typha is up, we need to run Typha itself # as a host-networked pod. serviceAccountName: calico-node @@ -642,7 +645,8 @@ spec: selector: matchLabels: k8s-app: calico-typha -{{- end -}} +{{- end }} + --- # Source: calico/templates/calico-node.yaml # This manifest installs the calico-node container, as well @@ -766,12 +770,14 @@ spec: # Use Kubernetes API as the backing datastore. - name: DATASTORE_TYPE value: "kubernetes" + {{- if .Networking.Calico.TyphaReplicas }} # Typha support: controlled by the ConfigMap. - name: FELIX_TYPHAK8SSERVICENAME valueFrom: configMapKeyRef: name: calico-config key: typha_service_name + {{- end }} # Wait for the datastore. - name: WAIT_FOR_DATASTORE value: "true" @@ -1089,4 +1095,4 @@ spec: path: "/etc/ssl/certs/ca-certificates.crt" nodeSelector: node-role.kubernetes.io/master: "" -{{- end -}} +{{ end -}} From 5c57ce49f9604f97ec00a2c46909fe1f782e7323 Mon Sep 17 00:00:00 2001 From: Ciprian Hacman Date: Sat, 14 Dec 2019 15:06:27 +0200 Subject: [PATCH 6/6] Revert ClusterRole name to "calico" --- .../networking.projectcalico.org.canal/k8s-1.15.yaml.template | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template index e2d3830c842fa..3cd722c58ffe9 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org.canal/k8s-1.15.yaml.template @@ -314,7 +314,7 @@ spec: kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: calico-node + name: calico labels: role.kubernetes.io/networking: "1" rules: @@ -465,7 +465,7 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: calico-node + name: calico subjects: - kind: ServiceAccount name: canal