Remove non-kubeadm deployment (kubernetes-sigs#3811)

* Remove non-kubeadm deployment * More cleanup * More cleanup * More cleanup * More cleanup * Fix gitlab * Try stop gce first before absent to make the delete process work * More cleanup * Fix bug with checking if kubeadm has already run * Fix bug with checking if kubeadm has already run * More fixes * Fix test * fix * Fix gitlab checkout untill kubespray 2.8 is on quay * Fixed * Add upgrade path from non-kubeadm to kubeadm. Revert ssl path * Readd secret checking * Do gitlab checks from v2.7.0 test upgrade path to 2.8.0 * fix typo * Fix CI jobs to kubeadm again. Fix broken hyperkube path * Fix gitlab * Fix rotate tokens * More fixes * More fixes * Fix tokens
reynencourt · Dec 6, 2018 · ddffdb6 · ddffdb6
1 parent 0d1be39
commit ddffdb6
Show file tree

Hide file tree

Showing 65 changed files with 111 additions and 2,042 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -24,7 +24,6 @@ variables:
   IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
   UPGRADE_TEST: "false"
-  KUBEADM_ENABLED: "false"
   LOG_LEVEL: "-vv"
 
 # asia-east1-a
@@ -89,11 +88,11 @@ before_script:
     - echo ${PWD}
     - echo "${STARTUP_SCRIPT}"
     - cd tests && make create-${CI_PLATFORM} -s ; cd -
+    #- git fetch --all && git checkout v2.7.0
 
     # Check out latest tag if testing upgrade
     # Uncomment when gitlab kubespray repo has tags
-    #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
-    - test "${UPGRADE_TEST}" != "false" && git checkout 53d87e53c5899d4ea2904ab7e3883708dd6363d3
+    - test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
     # Checkout the CI vars file so it is available
     - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
     # Workaround https://github.com/kubernetes-sigs/kubespray/issues/2021
@@ -137,9 +136,7 @@ before_script:
 
     # Tests Cases
     ## Test Master API
-    - >
-      ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
-      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+    - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
 
     ## Ping the between 2 pod
     - ansible-playbook -i ${ANSIBLE_INVENTORY} -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL

diff --git a/cluster.yml b/cluster.yml
@@ -13,20 +13,6 @@
   vars:
     ansible_connection: local
 
-- hosts: localhost
-  gather_facts: false
-  tasks:
-    - name: deploy warning for non kubeadm
-      debug:
-        msg: "DEPRECATION: non-kubeadm deployment is deprecated from v2.9. Will be removed in next release."
-      when: not kubeadm_enabled and not skip_non_kubeadm_warning
-
-    - name: deploy cluster for non kubeadm
-      pause:
-        prompt: "Are you sure you want to deploy cluster using the deprecated non-kubeadm mode."
-        echo: no
-      when: not kubeadm_enabled and not skip_non_kubeadm_warning
-
 - hosts: bastion[0]
   gather_facts: False
   roles:
@@ -96,15 +82,15 @@
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}
-    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+    - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: network_plugin, tags: network }
 
 - hosts: kube-master[0]
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
-    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"], when: "kubeadm_enabled" }
+    - { role: win_nodes/kubernetes_patch, tags: ["master", "win_nodes"]}
 
 - hosts: kube-master
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"

diff --git a/contrib/dind/kubespray-dind.yaml b/contrib/dind/kubespray-dind.yaml
@@ -1,7 +1,6 @@
 # kubespray-dind.yaml: minimal kubespray ansible playbook usable for DIND
 # See contrib/dind/README.md
 kube_api_anonymous_auth: true
-kubeadm_enabled: true
 
 kubelet_fail_swap_on: false
 

diff --git a/contrib/dind/test-some_distros-kube_router_combo.env b/contrib/dind/test-some_distros-kube_router_combo.env
@@ -1,8 +1,6 @@
 DISTROS=(debian centos)
 NETCHECKER_HOST=${NODES[0]}
 EXTRAS=(
-  'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":false}'
-  'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":true}'
-  'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":false}'
-  'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":true}'
+  'kube_network_plugin=kube-router {"kube_router_run_service_proxy":false}'
+  'kube_network_plugin=kube-router {"kube_router_run_service_proxy":true}'
 )
diff --git a/contrib/dind/test-some_distros-most_CNIs.env b/contrib/dind/test-some_distros-most_CNIs.env
@@ -1,8 +1,8 @@
 DISTROS=(debian centos)
 EXTRAS=(
-  'kube_network_plugin=calico {"kubeadm_enabled":true}'
-  'kube_network_plugin=canal {"kubeadm_enabled":true}'
-  'kube_network_plugin=cilium {"kubeadm_enabled":true}'
-  'kube_network_plugin=flannel {"kubeadm_enabled":true}'
-  'kube_network_plugin=weave {"kubeadm_enabled":true}'
+  'kube_network_plugin=calico {}'
+  'kube_network_plugin=canal {}'
+  'kube_network_plugin=cilium {}'
+  'kube_network_plugin=flannel {}'
+  'kube_network_plugin=weave {}'
 )
diff --git a/docs/cri-o.md b/docs/cri-o.md
@@ -15,8 +15,6 @@ Use cri-o instead of docker, set following variable:
 #### all.yml
 
 ```
-kubeadm_enabled: true
-...
 download_container: false
 skip_downloads: false
 ```
@@ -28,4 +26,3 @@ etcd_deployment_type: host
 kubelet_deployment_type: host
 container_manager: crio
 ```
-
diff --git a/docs/kube-router.md b/docs/kube-router.md
@@ -62,34 +62,6 @@ You can change the default configuration by overriding `kube_router_...` variabl
 these are named to follow `kube-router` command-line options as per
 <https://www.kube-router.io/docs/user-guide/#try-kube-router-with-cluster-installers>.
 
-## Caveats
-
-### kubeadm_enabled: true
-
-If you want to set `kube-router` to replace `kube-proxy`
-(`--run-service-proxy=true`) while using `kubeadm_enabled`,
-then 'kube-proxy` DaemonSet will be removed *after* kubeadm finishes
-running, as it's not possible to skip kube-proxy install in kubeadm flags
-and/or config, see https://github.com/kubernetes/kubeadm/issues/776.
-
-Given above, if `--run-service-proxy=true` is needed it would be
-better to void `kubeadm_enabled` i.e. set:
-
-```
-kubeadm_enabled: false
-kube_router_run_service_proxy: true
-
-```
-
-If for some reason you do want/need to set `kubeadm_enabled`, removing
-it afterwards behave better if kube-proxy is set to ipvs mode, i.e. set:
-
-```
-kubeadm_enabled: true
-kube_router_run_service_proxy: true
-kube_proxy_mode: ipvs
-```
-
 ## Advanced BGP Capabilities
 https://github.com/cloudnativelabs/kube-router#advanced-bgp-capabilities
 
@@ -105,4 +77,4 @@ Next options will set up annotations for kube-router, using `kubectl annotate` c
 kube_router_annotations_master: []
 kube_router_annotations_node: []
 kube_router_annotations_all: []
-```
+```
diff --git a/docs/vars.md b/docs/vars.md
@@ -20,9 +20,6 @@ Some variables of note include:
   string)
 * *etcd_version* - Specify version of ETCD to use
 * *ipip* - Enables Calico ipip encapsulation by default
-* *hyperkube_image_repo* - Specify the Docker repository where Hyperkube
-  resides
-* *hyperkube_image_tag* - Specify the Docker tag where Hyperkube resides
 * *kube_network_plugin* - Sets k8s network plugin (default Calico)
 * *kube_proxy_mode* - Changes k8s proxy mode to iptables mode
 * *kube_version* - Specify a given Kubernetes hyperkube version

diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml
@@ -42,16 +42,10 @@ bin_dir: /usr/local/bin
 ## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external'
 ## When openstack is used make sure to source in the openstack credentials
 ## like you would do when using nova-client before starting the playbook.
-## Note: The 'external' cloud provider is not supported. 
+## Note: The 'external' cloud provider is not supported.
 ## TODO(riverzhang): https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#running-cloud-controller-manager
 #cloud_provider:
 
-## kubeadm deployment mode
-kubeadm_enabled: true
-
-# Skip alert information
-skip_non_kubeadm_warning: false
-
 ## Set these proxy values in order to update package manager and docker daemon to use proxies
 #http_proxy: ""
 #https_proxy: ""

diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
@@ -136,8 +136,6 @@ calico_policy_image_repo: "quay.io/calico/kube-controllers"
 calico_policy_image_tag: "{{ calico_policy_version }}"
 calico_rr_image_repo: "quay.io/calico/routereflector"
 calico_rr_image_tag: "{{ calico_rr_version }}"
-hyperkube_image_repo: "{{ kube_image_repo }}/hyperkube-{{ image_arch }}"
-hyperkube_image_tag: "{{ kube_version }}"
 pod_infra_image_repo: "gcr.io/google_containers/pause-{{ image_arch }}"
 pod_infra_image_tag: "{{ pod_infra_version }}"
 install_socat_image_repo: "xueshanf/install-socat"
@@ -272,7 +270,7 @@ downloads:
       - k8s-cluster
 
   kubeadm:
-    enabled: "{{ kubeadm_enabled }}"
+    enabled: true
     file: true
     version: "{{ kubeadm_version }}"
     dest: "{{local_release_dir}}/kubeadm"
@@ -284,20 +282,11 @@ downloads:
     groups:
       - k8s-cluster
 
-  hyperkube:
-    enabled: "{{ kubeadm_enabled == false }}"
-    container: true
-    repo: "{{ hyperkube_image_repo }}"
-    tag: "{{ hyperkube_image_tag }}"
-    sha256: "{{ hyperkube_digest_checksum|default(None) }}"
-    groups:
-      - k8s-cluster
-
   hyperkube_file:
     enabled: true
     file: true
     version: "{{ kube_version }}"
-    dest: "{{local_release_dir}}/hyperkube"
+    dest: "{{ local_release_dir }}/hyperkube"
     sha256: "{{ hyperkube_binary_checksum }}"
     url: "{{ hyperkube_download_url }}"
     unarchive: false

diff --git a/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml b/roles/kubernetes-apps/ansible/tasks/cleanup_dns.yml
@@ -21,7 +21,6 @@
     resource: "deploy"
     state: absent
   when:
-    - kubeadm_enabled|default(false)
     - kubeadm_init is defined
     - kubeadm_init.changed|default(false)
     - inventory_hostname == groups['kube-master'][0]
@@ -50,7 +49,6 @@
     - 'deploy'
     - 'svc'
   when:
-    - kubeadm_enabled|default(false)
     - kubeadm_init is defined
     - kubeadm_init.changed|default(false)
     - inventory_hostname == groups['kube-master'][0]

diff --git a/roles/kubernetes-apps/rotate_tokens/tasks/main.yml b/roles/kubernetes-apps/rotate_tokens/tasks/main.yml
@@ -1,14 +1,14 @@
 ---
 - name: Rotate Tokens | Get default token name
-  shell: "{{ bin_dir }}/kubectl get secrets -o custom-columns=name:{.metadata.name} --no-headers | grep -m1 default-token"
+  shell: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets -o custom-columns=name:{.metadata.name} --no-headers | grep -m1 default-token"
   register: default_token
   changed_when: false
   until: default_token.rc == 0
   delay: 1
   retries: 5
 
 - name: Rotate Tokens | Get default token data
-  command: "{{ bin_dir }}/kubectl get secrets {{ default_token.stdout }} -ojson"
+  command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets {{ default_token.stdout }} -ojson"
   register: default_token_data
   changed_when: false
 
@@ -31,18 +31,18 @@
 # instead of filtering manually
 - name: Rotate Tokens | Get all serviceaccount tokens to expire
   shell: >-
-    {{ bin_dir }}/kubectl get secrets --all-namespaces
+    {{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf get secrets --all-namespaces
     -o 'jsonpath={range .items[*]}{"\n"}{.metadata.namespace}{" "}{.metadata.name}{" "}{.type}{end}'
     | grep kubernetes.io/service-account-token
     | egrep 'default-token|kube-proxy|kube-dns|dnsmasq|netchecker|weave|calico|canal|flannel|dashboard|cluster-proportional-autoscaler|tiller|local-volume-provisioner'
   register: tokens_to_delete
   when: needs_rotation
 
 - name: Rotate Tokens | Delete expired tokens
-  command: "{{ bin_dir }}/kubectl delete secrets -n {{ item.split(' ')[0] }} {{ item.split(' ')[1] }}"
+  command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete secrets -n {{ item.split(' ')[0] }} {{ item.split(' ')[1] }}"
   with_items: "{{ tokens_to_delete.stdout_lines }}"
   when: needs_rotation
 
 - name: Rotate Tokens | Delete pods in system namespace
-  command: "{{ bin_dir }}/kubectl delete pods -n kube-system --all"
+  command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete pods -n kube-system --all"
   when: needs_rotation
diff --git a/roles/kubernetes/client/tasks/main.yml b/roles/kubernetes/client/tasks/main.yml
@@ -10,25 +10,6 @@
   tags:
     - facts
 
-- name: Gather certs for admin kubeconfig
-  slurp:
-    src: "{{ item }}"
-  register: admin_certs
-  with_items:
-    - "{{ kube_cert_dir }}/ca.pem"
-    - "{{ kube_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    - "{{ kube_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-  when: not kubeadm_enabled|d(false)|bool
-
-- name: Write admin kubeconfig
-  template:
-    src: admin.conf.j2
-    dest: "{{ kube_config_dir }}/admin.conf"
-    owner: root
-    group: "{{ kube_cert_group }}"
-    mode: 0640
-  when: not kubeadm_enabled|d(false)|bool
-
 - name: Create kube config dir
   file:
     path: "/root/.kube"

diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml
@@ -124,7 +124,7 @@
 # FIXME(jjo): need to post-remove kube-proxy until https://github.com/kubernetes/kubeadm/issues/776
 # is fixed
 - name: Delete kube-proxy daemonset if kube_proxy_remove set, e.g. kube_network_plugin providing proxy services
-  shell: "{{ bin_dir }}/kubectl delete daemonset -n kube-system kube-proxy"
+  shell: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete daemonset -n kube-system kube-proxy"
   delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
   when:

diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml
@@ -91,13 +91,16 @@
   command: /bin/true
   notify:
     - Master | set secret_changed to true
-    - Master | clear kubeconfig for root user
+    - Master | Copy new kubeconfig for root user
 
 - name: Master | set secret_changed to true
   set_fact:
     secret_changed: true
 
-- name: Master | clear kubeconfig for root user
-  file:
-    path: /root/.kube/config
-    state: absent
+- name: Master | Copy new kubeconfig for root user
+  copy:
+    src: "{{ kube_config_dir }}/admin.conf"
+    dest: "/root/.kube/config"
+    remote_src: yes
+    mode: "0600"
+    backup: yes
diff --git a/roles/kubernetes/master/tasks/kubeadm-cleanup-old-certs.yml b/roles/kubernetes/master/tasks/kubeadm-cleanup-old-certs.yml
@@ -1,7 +1,7 @@
 ---
 - name: kubeadm | Retrieve files to purge
   find:
-    paths: "{{kube_cert_dir }}"
+    paths: "{{ kube_cert_dir }}"
     patterns: '*.pem'
   register: files_to_purge_for_kubeadm
 

diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml
@@ -26,19 +26,22 @@
   file:
     path: "{{ kube_config_dir }}/admin.conf"
     state: absent
-  when: not kubeadm_already_run.stat.exists
+  when:
+    - not kubeadm_already_run.stat.exists
 
 - name: kubeadm | Delete old static pods
   file:
     path: "{{ kube_config_dir }}/manifests/{{item}}.manifest"
     state: absent
   with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler", "kube-proxy"]
-  when: old_apiserver_cert.stat.exists
+  when:
+    - old_apiserver_cert.stat.exists
 
 - name: kubeadm | Forcefully delete old static pods
   shell: "docker ps -f name=k8s_{{item}} -q | xargs --no-run-if-empty docker rm -f"
   with_items: ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
-  when: old_apiserver_cert.stat.exists
+  when:
+    - old_apiserver_cert.stat.exists
 
 - name: kubeadm | aggregate all SANs
   set_fact:
@@ -220,7 +223,8 @@
 
 - name: kubeadm | cleanup old certs if necessary
   import_tasks: kubeadm-cleanup-old-certs.yml
-  when: old_apiserver_cert.stat.exists
+  when:
+    - old_apiserver_cert.stat.exists
 
 - name: kubeadm | Remove taint for master with node role
   command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"