Merge branch 'master' of https://github.com/kubernetes-sigs/kubespray

* 'master' of https://github.com/kubernetes-sigs/kubespray:
  change owner to root for bin_dir directory (kubernetes-sigs#6814)
  Modify imagepullpolicy (kubernetes-sigs#6816)
  fix: add tags for set facts nodelocaldns (kubernetes-sigs#6813)
  Make reset work for crio (kubernetes-sigs#6812)
  Added option to force apiserver and respective client certificate to … (kubernetes-sigs#6403)
  cleanup kubelet_deployment_type (kubernetes-sigs#6815)
  allow pre-existing floating IPs to be specified with k8s_master_fips (kubernetes-sigs#6755)
  Fix line-spacing in no_proxy.yml (kubernetes-sigs#6810)
  Fix handler naming issue for Kubeadm | kubelet (kubernetes-sigs#6803)
  Disable dashboard by default (kubernetes-sigs#6804)
  Chmod kubeconfig to avoid group-readable (kubernetes-sigs#6800)
  Update bunch of dependencies (kubernetes-sigs#6801)
  If no_proxy_exclude_workers is true, workers will be excluded from the no_proxy variable.  This prevents docker engine restarting when scaling workers. (kubernetes-sigs#6520)

README.md

@@ -123,7 +123,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [cri-o](http://cri-o.io/) v1.17 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v0.8.7
-  - [calico](https://github.com/projectcalico/calico) v3.16.1
+  - [calico](https://github.com/projectcalico/calico) v3.16.2
   - [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
   - [cilium](https://github.com/cilium/cilium) v1.8.4
   - [contiv](https://github.com/contiv/install) v1.2.1
@@ -139,7 +139,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
   - [cert-manager](https://github.com/jetstack/cert-manager) v0.16.1
   - [coredns](https://github.com/coredns/coredns) v1.7.0
-  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.40.1
+  - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.40.2
 
 Note: The list of validated [docker versions](https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker) is 1.13.1, 17.03, 17.06, 17.09, 18.06, 18.09 and 19.03. The recommended docker version is 19.03. The kubelet might break on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin).
 

contrib/terraform/openstack/README.md

@@ -239,6 +239,7 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
 |`network_dns_domain` | (Optional) The dns_domain for the internal network that will be generated |
 |`dns_nameservers`| An array of DNS name server names to be used by hosts in the internal subnet. |
 |`floatingip_pool` | Name of the pool from which floating IPs will be allocated |
+|`k8s_master_fips` | A list of floating IPs that you have already pre-allocated; they will be attached to master nodes instead of creating new random floating IPs. |
 |`external_net` | UUID of the external network that will be routed to |
 |`flavor_k8s_master`,`flavor_k8s_node`,`flavor_etcd`, `flavor_bastion`,`flavor_gfs_node` | Flavor depends on your openstack installation, you can get available flavor IDs through `openstack flavor list` |
 |`image`,`image_gfs` | Name of the image to use in provisioning the compute resources. Should already be loaded into glance. |

contrib/terraform/openstack/kubespray.tf

@@ -27,6 +27,7 @@ module "ips" {
   network_name                  = var.network_name
   router_id                     = module.network.router_id
   k8s_nodes                     = var.k8s_nodes
+  k8s_master_fips               = var.k8s_master_fips
 }
 
 module "compute" {

contrib/terraform/openstack/modules/ips/main.tf

@@ -4,14 +4,16 @@ resource "null_resource" "dummy_dependency" {
   }
 }
 
+# If user specifies pre-existing IPs to use in k8s_master_fips, do not create new ones.
 resource "openstack_networking_floatingip_v2" "k8s_master" {
-  count      = var.number_of_k8s_masters
+  count      = length(var.k8s_master_fips) > 0 ? 0 : var.number_of_k8s_masters
   pool       = var.floatingip_pool
   depends_on = [null_resource.dummy_dependency]
 }
 
+# If user specifies pre-existing IPs to use in k8s_master_fips, do not create new ones.
 resource "openstack_networking_floatingip_v2" "k8s_master_no_etcd" {
-  count      = var.number_of_k8s_masters_no_etcd
+  count      = length(var.k8s_master_fips) > 0 ? 0 : var.number_of_k8s_masters_no_etcd
   pool       = var.floatingip_pool
   depends_on = [null_resource.dummy_dependency]
 }

contrib/terraform/openstack/modules/ips/outputs.tf

@@ -1,9 +1,11 @@
+# If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created.
 output "k8s_master_fips" {
-  value = openstack_networking_floatingip_v2.k8s_master[*].address
+  value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master[*].address
 }
 
+# If k8s_master_fips is already defined as input, keep the same value since new FIPs have not been created.
 output "k8s_master_no_etcd_fips" {
-  value = openstack_networking_floatingip_v2.k8s_master_no_etcd[*].address
+  value = length(var.k8s_master_fips) > 0 ? var.k8s_master_fips : openstack_networking_floatingip_v2.k8s_master_no_etcd[*].address
 }
 
 output "k8s_node_fips" {

contrib/terraform/openstack/modules/ips/variables.tf

@@ -17,3 +17,5 @@ variable "router_id" {
 }
 
 variable "k8s_nodes" {}
+
+variable "k8s_master_fips" {}

contrib/terraform/openstack/variables.tf

@@ -156,6 +156,12 @@ variable "dns_nameservers" {
   default     = []
 }
 
+variable "k8s_master_fips" {
+  description = "specific pre-existing floating IPs to use for master nodes"
+  type        = list(string)
+  default     = []
+}
+
 variable "floatingip_pool" {
   description = "name of the floating ip pool to use"
   default     = "external"

docs/cri-o.md

@@ -19,7 +19,6 @@ skip_downloads: false
 ## k8s-cluster.yml
 
 ```yaml
-kubelet_deployment_type: host
 container_manager: crio
 ```
 

docs/proxy.md

@@ -14,3 +14,10 @@ If you set http and https proxy, all nodes and loadbalancer will be excluded fro
 ## Set additional addresses to default no_proxy (all cluster nodes and loadbalancer)
 
 `additional_no_proxy: "aditional_host,"`
+
+## Exclude workers from no_proxy
+
+Since workers are included in the no_proxy variable, by default, docker engine will be restarted on all nodes (all
+pods will restart) when adding or removing workers.  To override this behaviour by only including master nodes in the
+no_proxy variable, set:
+`no_proxy_exclude_workers: true`

docs/vars.md

@@ -109,14 +109,9 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
 * *docker_plugins* - This list can be used to define [Docker plugins](https://docs.docker.com/engine/extend/) to install.
 * *containerd_config* - Controls some parameters in containerd configuration file (usually /etc/containerd/config.toml).
   [Default config](https://github.com/kubernetes-sigs/kubespray/blob/master/roles/container-engine/containerd/defaults/main.yml) can be overriden in inventory vars.
-* *http_proxy/https_proxy/no_proxy* - Proxy variables for deploying behind a
+* *http_proxy/https_proxy/no_proxy/no_proxy_exclude_workers/additional_no_proxy* - Proxy variables for deploying behind a
   proxy. Note that no_proxy defaults to all internal cluster IPs and hostnames
   that correspond to each node.
-* *kubelet_deployment_type* - Controls which platform to deploy kubelet on.
-  Available options are ``host`` and ``docker``. ``docker`` mode
-  is unlikely to work on newer releases. Starting with Kubernetes v1.7
-  series, this now defaults to ``host``. Before v1.7, the default was Docker.
-  This is because of cgroup [issues](https://github.com/kubernetes/kubernetes/issues/43704).
 * *kubelet_cgroup_driver* - Allows manual override of the
   cgroup-driver option for Kubelet. By default autodetection is used
   to match Docker configuration.

inventory/sample/group_vars/all/all.yml

@@ -68,6 +68,11 @@ loadbalancer_apiserver_healthcheck_port: 8081
 ## If you need exclude all cluster nodes from proxy and other resources, add other resources here.
 # additional_no_proxy: ""
 
+## Since workers are included in the no_proxy variable by default, docker engine will be restarted on all nodes (all
+## pods will restart) when adding or removing workers.  To override this behaviour by only including master nodes in the
+## no_proxy variable, set below to true:
+no_proxy_exclude_workers: false
+
 ## Certificate Management
 ## This setting determines whether certs are generated via scripts.
 ## Chose 'none' if you provide your own certificates.

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -204,8 +204,6 @@ kata_containers_enabled: false
 # containerd_untrusted_runtime_engine: ''
 # containerd_untrusted_runtime_root: ''
 
-## Settings for containerized control plane (kubelet/secrets)
-kubelet_deployment_type: host
 helm_deployment_type: host
 
 kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}"
@@ -314,3 +312,5 @@ persistent_volumes_enabled: false
 
 ## Amount of time to retain events. (default 1h0m0s)
 event_ttl_duration: "1h0m0s"
+##  Force regeneration of kubernetes control plane certificates without the need of bumping the cluster version
+force_certificate_regeneration: false

roles/bootstrap-os/files/bootstrap.sh

@@ -2,7 +2,7 @@
 set -e
 
 BINDIR="/opt/bin"
-PYPY_VERSION=7.3.1
+PYPY_VERSION=7.3.2
 PYPI_URL="https://downloads.python.org/pypy/pypy3.6-v${PYPY_VERSION}-linux64.tar.bz2"
 PYPI_HASH=f67cf1664a336a3e939b58b3cabfe47d893356bdc01f2e17bc912aaa6605db12
 

roles/download/defaults/main.yml

@@ -65,7 +65,7 @@ quay_image_repo: "quay.io"
 
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
-calico_version: "v3.16.1"
+calico_version: "v3.16.2"
 calico_ctl_version: "{{ calico_version }}"
 calico_cni_version: "{{ calico_version }}"
 calico_policy_version: "{{ calico_version }}"
@@ -360,13 +360,13 @@ cni_binary_checksums:
   amd64: 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8
 calicoctl_binary_checksums:
   arm:
-    v3.16.1: 0
+    v3.16.2: 0
     v3.15.2: 0
   amd64:
-    v3.16.1: 7c33a841fdf85409c2eee5b287e1212d6c7e82885ec9ffaf690b6019b7b80c1b
+    v3.16.2: 801b059a4fd0dac8795693026c69a79a00dd2353eff597cc36b79fcb6ec53a0a
     v3.15.2: 219ae954501cbe15daeda0ad52e13ec65f99c77548c7d3cbfc4ced5c7149fdf1
   arm64:
-    v3.16.1: d3cc8b721a862f0c50273706bf6d38e47ee9b932b8d90a0f0e51280594a6f242
+    v3.16.2: aa5695940ec8a36393725a5ce7b156f776fed8da38b994c0828d7f3a60e59bc6
     v3.15.2: 49165f9e4ad55402248b578310fcf68a57363f54e66be04ac24be9714899b4d5
 
 etcd_binary_checksum: "{{ etcd_binary_checksums[image_arch] }}"
@@ -441,16 +441,16 @@ ovn4nfv_k8s_plugin_image_tag: "{{ ovn4nfv_k8s_plugin_image_version }}"
 nginx_image_repo: "{{ docker_image_repo }}/library/nginx"
 nginx_image_tag: 1.19
 haproxy_image_repo: "{{ docker_image_repo }}/library/haproxy"
-haproxy_image_tag: 2.1
+haproxy_image_tag: 2.2
 
 # Coredns version should be supported by corefile-migration (or at least work with)
 # bundle with kubeadm; if not 'basic' upgrade can sometimes fail
 coredns_version: "1.7.0"
 coredns_image_repo: "{{ docker_image_repo }}/coredns/coredns"
 coredns_image_tag: "{{ coredns_version }}"
 
-nodelocaldns_version: "1.15.13"
-nodelocaldns_image_repo: "{{ kube_image_repo }}/k8s-dns-node-cache"
+nodelocaldns_version: "1.15.14"
+nodelocaldns_image_repo: "{{ kube_image_repo }}/dns/k8s-dns-node-cache"
 nodelocaldns_image_tag: "{{ nodelocaldns_version }}"
 
 dnsautoscaler_version: 1.8.3
@@ -482,7 +482,7 @@ rbd_provisioner_image_tag: "v2.1.1-k8s1.11"
 local_path_provisioner_image_repo: "{{ docker_image_repo }}/rancher/local-path-provisioner"
 local_path_provisioner_image_tag: "v0.0.17"
 ingress_nginx_controller_image_repo: "{{ kube_image_repo }}/ingress-nginx/controller"
-ingress_nginx_controller_image_tag: "v0.40.1"
+ingress_nginx_controller_image_tag: "v0.40.2"
 ingress_ambassador_image_repo: "{{ quay_image_repo }}/datawire/ambassador-operator"
 ingress_ambassador_image_tag: "v1.2.9"
 alb_ingress_image_repo: "{{ docker_image_repo }}/amazon/aws-alb-ingress-controller"

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -43,10 +43,9 @@ netchecker_agent_group: 1000
 netchecker_server_group: 1000
 
 # Dashboard
-dashboard_enabled: true
 dashboard_replicas: 1
 
-# Namespace for dashboad
+# Namespace for dashboard
 dashboard_namespace: kube-system
 
 # Limits for dashboard

roles/kubernetes-apps/ansible/tasks/nodelocaldns.yml

@@ -8,6 +8,12 @@
       {{ manual_dns_server }}
       {%- endif -%}
     secondaryclusterIP: "{{ skydns_server_secondary }}"
+  when:
+    - enable_nodelocaldns
+    - inventory_hostname == groups['kube-master'] | first
+  tags:
+    - nodelocaldns
+    - coredns
 
 - name: Kubernetes Apps | Lay Down nodelocaldns Template
   template:

roles/kubernetes-apps/ingress_controller/ambassador/templates/deploy-ambassador.yml.j2

@@ -29,7 +29,7 @@ spec:
           image: {{ ingress_ambassador_image_repo }}:{{ ingress_ambassador_image_tag }}
           command:
           - ambassador-operator
-          imagePullPolicy: Always
+          imagePullPolicy: {{ k8s_image_pull_policy }}
           env:
             - name: WATCH_NAMESPACE
               valueFrom:

roles/kubernetes-apps/metallb/templates/metallb.yml.j2

@@ -318,7 +318,7 @@ spec:
               name: memberlist
               key: secretkey
         image: {{ metallb_speaker_image_repo }}:{{ metallb_version }}
-        imagePullPolicy: Always
+        imagePullPolicy: {{ k8s_image_pull_policy }}
         name: speaker
         ports:
         - containerPort: {{ metallb_port }}
@@ -374,7 +374,7 @@ spec:
         - --port={{ metallb_port }}
         - --config=config
         image: {{ metallb_controller_image_repo }}:{{ metallb_version }}
-        imagePullPolicy: Always
+        imagePullPolicy: {{ k8s_image_pull_policy }}
         name: controller
         ports:
         - containerPort: {{ metallb_port }}

roles/kubernetes/client/tasks/main.yml

@@ -87,7 +87,7 @@
   copy:
     content: "{{ final_admin_kubeconfig | to_nice_yaml(indent=2) }}"
     dest: "{{ artifacts_dir }}/admin.conf"
-    mode: 0640
+    mode: 0600
   delegate_to: localhost
   connection: local
   become: no

roles/kubernetes/kubeadm/handlers/main.yml

@@ -3,13 +3,13 @@
   command: /bin/true
   notify:
     - Kubeadm | reload systemd
-    - Kubeadm | restart kubelet
+    - Kubeadm | reload kubelet
 
 - name: Kubeadm | reload systemd
   systemd:
     daemon_reload: true
 
-- name: Kubeadm | restart kubelet
+- name: Kubeadm | reload kubelet
   service:
     name: kubelet
     state: restarted

roles/kubernetes/master/defaults/main/main.yml

@@ -193,3 +193,5 @@ secrets_encryption_query: "resources[*].providers[0].{{kube_encryption_algorithm
 
 ## Amount of time to retain events. (default 1h0m0s)
 event_ttl_duration: "1h0m0s"
+##  Force regeneration of kubernetes control plane certificates without the need of bumping the cluster version
+force_certificate_regeneration: false

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -130,7 +130,7 @@
   when:
     - inventory_hostname == groups['kube-master']|first
     - kubeadm_already_run.stat.exists
-    - apiserver_sans_check.changed
+    - apiserver_sans_check.changed or force_certificate_regeneration
 
 - name: kubeadm | regenerate apiserver cert 2/2
   command: >-
@@ -140,7 +140,7 @@
   when:
     - inventory_hostname == groups['kube-master']|first
     - kubeadm_already_run.stat.exists
-    - apiserver_sans_check.changed
+    - apiserver_sans_check.changed or force_certificate_regeneration
 
 - name: kubeadm | Initialize first master
   command: >-

roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -17,13 +17,32 @@
     - master
     - node
   with_items:
-    - "{{ bin_dir }}"
     - "{{ kube_config_dir }}"
     - "{{ kube_cert_dir }}"
     - "{{ kube_manifest_dir }}"
     - "{{ kube_script_dir }}"
     - "{{ kubelet_flexvolumes_plugins_dir }}"
 
+- name: Create other directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: root
+  when: inventory_hostname in groups['k8s-cluster']
+  become: true
+  tags:
+    - kubelet
+    - k8s-secrets
+    - kube-controller-manager
+    - kube-apiserver
+    - bootstrap-os
+    - apps
+    - network
+    - master
+    - node
+  with_items:
+    - "{{ bin_dir }}"
+
 - name: Check if kubernetes kubeadm compat cert dir exists
   stat:
     path: "{{ kube_cert_compat_dir }}"

roles/kubespray-defaults/defaults/main.yaml

@@ -314,7 +314,7 @@ k8s_image_pull_policy: IfNotPresent
 
 # Kubernetes dashboard
 # RBAC required. see docs/getting-started.md for access details.
-dashboard_enabled: true
+dashboard_enabled: false
 
 # Addons which can be enabled
 helm_enabled: false

roles/kubespray-defaults/tasks/no_proxy.yml

@@ -6,7 +6,12 @@
       {{ apiserver_loadbalancer_domain_name| default('') }},
       {{ loadbalancer_apiserver.address | default('') }},
       {%- endif -%}
-      {%- for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}
+      {%- if ( (no_proxy_exclude_workers is defined) and (no_proxy_exclude_workers) ) -%}
+      {% set cluster_or_master = 'kube-master' %}
+      {% else %}
+      {% set cluster_or_master = 'k8s-cluster' %}
+      {% endif %}
+      {%- for item in (groups[cluster_or_master] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}
       {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }},
       {%-   if item != hostvars[item].get('ansible_hostname', '') -%}
       {{ hostvars[item]['ansible_hostname'] }},