From add030972cafd9eab4582e87a2a848492246830b Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Thu, 6 May 2021 01:45:46 +0000 Subject: [PATCH 1/4] Sync requirements.txt with Kubspray upstream --- requirements.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index f7753ea2..e7704edd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,11 @@ -# from kubespray v2.13.1 -ansible==2.10.0 +# from kubespray v2.15.1 +ansible==2.9.20 +cryptography==2.8 jinja2==2.11.3 netaddr==0.7.19 pbr==5.4.4 -hvac==0.10.0 jmespath==0.9.5 ruamel.yaml==0.16.10 # for taco docker -cryptography From f82b897bcae08feb74a472c95ebcba7aa15bc15d Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Thu, 6 May 2021 04:30:40 +0000 Subject: [PATCH 2/4] add rook operator and cluster deployment role --- defaults/TACOFIXME.yml | 2 +- defaults/global_taco.yml | 2 +- reset.yml | 5 ---- roles/ceph/rook/defaults/main.yml | 5 ++++ roles/ceph/rook/tasks/main.yml | 49 +++++++++++++++++++++++++++++++ site.yml | 12 +------- 6 files changed, 57 insertions(+), 18 deletions(-) create mode 100644 roles/ceph/rook/defaults/main.yml create mode 100644 roles/ceph/rook/tasks/main.yml diff --git a/defaults/TACOFIXME.yml b/defaults/TACOFIXME.yml index a9b7b6e7..c31210af 100644 --- a/defaults/TACOFIXME.yml +++ b/defaults/TACOFIXME.yml @@ -15,7 +15,7 @@ ceph_admin_keyring: >- {%- if groups["osds"] is defined -%}{%- else -%}TACOFIXME{%- endif -%} taco_storageclass_name: >- - {%- if taco_storage_backend == 'ceph' -%}rbd{%- elif taco_storage_backend == 'nfs' -%}nfs-client{%- else -%}TACOFIXME{%- endif -%} + {%- if taco_storage_backend == 'rook-ceph' -%}rbd{%- elif taco_storage_backend == 'nfs' -%}nfs-client{%- else -%}TACOFIXME{%- endif -%} # when deploying new ceph monitor_interface: TACOFIXME diff --git a/defaults/global_taco.yml b/defaults/global_taco.yml index dd1d6098..70dd23be 100644 --- a/defaults/global_taco.yml +++ b/defaults/global_taco.yml @@ -20,7 +20,7 @@ k8s_binary_repo_url: "" ceph_repo_url: "" # Storage configurations: ceph or nfs -taco_storage_backend: "ceph" +taco_storage_backend: "rook-ceph" # Taco Apps to deploy using Armada taco_apps: ["openstack"] diff --git a/reset.yml b/reset.yml index 7021cf02..a3db2a37 100644 --- a/reset.yml +++ b/reset.yml @@ -10,8 +10,3 @@ # Reset K8S cluster - include: kubespray/reset.yml tags: k8s - -# Purge Ceph cluster -- include: ceph-ansible/infrastructure-playbooks/purge-cluster.yml - tags: ceph - when: taco_storage_backend == 'ceph' diff --git a/roles/ceph/rook/defaults/main.yml b/roles/ceph/rook/defaults/main.yml new file mode 100644 index 00000000..235b942a --- /dev/null +++ b/roles/ceph/rook/defaults/main.yml @@ -0,0 +1,5 @@ +--- +rook_ceph_cluster_chart_source: "{{ lookup('env', 'HOME') }}/tacoplay/charts/taco-helm-charts/rook-ceph-cluster" +rook_ceph_cluster_mon_replicas: 3 +rook_ceph_cluster_taco_pool_size: 3 +rook_ceph_cluster_taco_pool_require_safe_size: "true" diff --git a/roles/ceph/rook/tasks/main.yml b/roles/ceph/rook/tasks/main.yml new file mode 100644 index 00000000..cc3c376d --- /dev/null +++ b/roles/ceph/rook/tasks/main.yml @@ -0,0 +1,49 @@ +--- +- name: create namespace for rook + shell: >- + {{ bin_dir }}/kubectl create namespace rook-ceph + ignore_errors: true + become: false + +- name: add rook release helm repository + shell: >- + {{ bin_dir }}/helm repo add rook-release https://charts.rook.io/release + become: false + +- name: install rook-operator chart + shell: >- + {{ bin_dir }}/helm install --namespace rook-ceph rook-ceph rook-release/rook-ceph + become: false + +- name: sleep for 60 seconds for rook-operator pod to be launched + wait_for: + timeout: 60 + +- name: wait for rook-operator pods become ready + shell: >- + {{ bin_dir }}/kubectl wait --namespace=rook-ceph --for=condition=Ready pods -l app={{ item }} --timeout=600s + become: false + delay: 10 + retries: 3 + with_items: + - rook-ceph-operator + +- name: install rook ceph cluster chart + shell: >- + {{ bin_dir }}/helm install --namespace rook-ceph rook-ceph-cluster {{ rook_ceph_cluster_chart_source }} \ + --set cluster.mon.count={{ rook_ceph_cluster_mon_replicas }} \ + --set block_pools[0].name=taco \ + --set block_pools[0].size={{ rook_ceph_cluster_taco_pool_size }} \ + --set block_pools[0].requireSafeReplicaSize={{ rook_ceph_cluster_taco_pool_require_safe_size }} + become: false + +- name: sleep for 300 seconds for rook ceph cluster to be initialized + wait_for: + timeout: 300 + +- name: wait for rook ceph cluster become ready + shell: >- + {{ bin_dir }}/kubectl wait -n rook-ceph --for=condition=Ready cephcluster rook-ceph --timeout=600s + become: false + delay: 10 + retries: 3 diff --git a/site.yml b/site.yml index 5dec5dbf..19006a65 100755 --- a/site.yml +++ b/site.yml @@ -33,17 +33,6 @@ - { role: kubespray/roles/container-engine, tags: container-registry, when: container_registry_enabled } - { role: container-registry/server, tags: container-registry, when: container_registry_enabled } -- name: install ceph - include: ceph-ansible/site.yml.sample - tags: ceph - when: taco_storage_backend == 'ceph' - -- name: post install for ceph - hosts: admin-node:kube-master - any_errors_fatal: "{{ any_errors_fatal | default(true) }}" - roles: - - { role: ceph/post-install, tags: ceph-post-install, when: taco_storage_backend == 'ceph' } - - name: install kubernetes include: kubespray/cluster.yml tags: k8s @@ -55,6 +44,7 @@ - { role: kubespray/roles/kubespray-defaults, tags: k8s-post-install } - { role: k8s/post-install, tags: k8s-post-install } - { role: k8s/clients, tags: k8s-post-install } + - { role: ceph/rook, tags: rook, when: taco_storage_backend == 'rook-ceph' } - { role: decapod, tags: decapod, when: decapod_enabled } - name: setup helm repository From d1225fb118a3feb02bb9bc71517ddfdb370c74ca Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Thu, 6 May 2021 04:40:20 +0000 Subject: [PATCH 3/4] remove ceph-ansible and rbd related resources --- .gitignore | 1 - VERSIONS | 1 - defaults/TACOFIXME.yml | 17 +------- defaults/global_ceph.yml | 44 --------------------- defaults/global_k8s-cluster.yml | 25 ------------ defaults/global_k8s-images.yml | 1 - defaults/global_taco.yml | 5 +-- include_defaults.yml | 6 --- roles/setup-os/conf-repos/defaults/main.yml | 1 - roles/setup-os/conf-repos/tasks/main.yml | 24 ----------- 10 files changed, 2 insertions(+), 123 deletions(-) delete mode 100644 defaults/global_ceph.yml diff --git a/.gitignore b/.gitignore index 77c784a2..d7523100 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ artifacts/ docker_registry/ /armada/ -ceph-ansible/ kubespray/ charts/ mirrors/ diff --git a/VERSIONS b/VERSIONS index 96f923ff..16c60e48 100644 --- a/VERSIONS +++ b/VERSIONS @@ -1,6 +1,5 @@ kubespray https://github.com/openinfradev/kubespray.git v2.15.1 #charts/openstack-helm https://github.com/openinfradev/openstack-helm.git master #charts/openstack-helm-infra https://github.com/openinfradev/openstack-helm-infra.git master -ceph-ansible https://github.com/openinfradev/ceph-ansible.git stable-4.0 charts/taco-helm-charts https://github.com/openinfradev/helm-charts.git main charts/argo-helm https://github.com/argoproj/argo-helm.git master diff --git a/defaults/TACOFIXME.yml b/defaults/TACOFIXME.yml index c31210af..2aad6852 100644 --- a/defaults/TACOFIXME.yml +++ b/defaults/TACOFIXME.yml @@ -3,6 +3,7 @@ # TACO + # Ceph # ceph_monitors and ceph_admin_keyring would be set automatically # from /etc/ceph/ceph.conf and ceph.client.admin.keyring @@ -17,11 +18,6 @@ ceph_admin_keyring: >- taco_storageclass_name: >- {%- if taco_storage_backend == 'rook-ceph' -%}rbd{%- elif taco_storage_backend == 'nfs' -%}nfs-client{%- else -%}TACOFIXME{%- endif -%} -# when deploying new ceph -monitor_interface: TACOFIXME -public_network: TACOFIXME -cluster_network: TACOFIXME - # K8S kube_pods_subnet: TACOFIXME kube_service_addresses: TACOFIXME @@ -36,17 +32,6 @@ ipip_mode: TACOFIXME # as number and border router IP should be set only if it is enabled peer_with_router: TACOFIXME -# RBD provisioner -# set below only when using existing ceph -rbd_provisioner_admin_id: >- - {%- if groups["osds"] is not defined -%}TACOFIXME{%- else -%}kube{%- endif -%} -rbd_provisioner_secret: >- - {%- if groups["osds"] is not defined -%}TACOFIXME{%- else -%}AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ=={%- endif -%} -rbd_provisioner_user_id: >- - {%- if groups["osds"] is not defined -%}TACOFIXME{%- else -%}kube{%- endif -%} -rbd_provisioner_user_secret: >- - {%- if groups["osds"] is not defined -%}TACOFIXME{%- else -%}AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ=={%- endif -%} - # external NFS server nfs_server: TACOFIXME nfs_path: TACOFIXME diff --git a/defaults/global_ceph.yml b/defaults/global_ceph.yml deleted file mode 100644 index 5983275a..00000000 --- a/defaults/global_ceph.yml +++ /dev/null @@ -1,44 +0,0 @@ -configure_firewall: false - -ceph_stable_release: nautilus -ceph_origin: repository -ceph_mirror: https://download.ceph.com - -ceph_repository: >- - {%- if ceph_repo_url != "" -%}custom{%- else -%}community{%- endif -%} - -# when using existing ceph -ceph_monitors: [] -ceph_admin_keyring: "" -external_cluster_mon_ips: "{{ ceph_monitors }}" - -#when using new ceph -monitor_interface: "" -public_network: "" -cluster_network: "" - -copy_admin_key: >- - {%- if (groups["mons"]|length>0) -%}true{%- else -%}false{%- endif -%} -ceph_mgr_modules: [status,dashboard,prometheus] - -cluster: ceph -ceph_conf_overrides: - global: - mon_allow_pool_delete: true - osd_pool_default_size: 3 - osd_pool_default_min_size: 2 - osd_pg_stat_report_internal_max: 1 - -osd_scenario: lvm -osd_objectstore: bluestore - -openstack_keys: - - { name: client.kube, caps: { mon: "profile rbd", osd: "profile rbd pool=kube"}, key: "AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ==", mode: "0600" } - - { name: client.cinder, caps: { mon: "profile rbd", osd: "profile rbd pool=volumes, profile rbd pool=backups, profile rbd pool=images"}, key: "AQAin8tU0CFgEhAATb7sYgtWsh+S5HEbg6MrGg==", mode: "0600" } - -dashboard_enabled: false - -# epel-release shouldn't be installed in offline env. -# In online env, it'll be installed by setup-os role anyway. -centos_package_dependencies: - - libselinux-python diff --git a/defaults/global_k8s-cluster.yml b/defaults/global_k8s-cluster.yml index 6290a648..61c7893d 100644 --- a/defaults/global_k8s-cluster.yml +++ b/defaults/global_k8s-cluster.yml @@ -17,30 +17,6 @@ calico_felix_prometheusmetricsport: 9091 calico_felix_prometheusgometricsenabled: "true" calico_felix_prometheusprocessmetricsenabled: "true" -# external provisioners > rbd_provisioner -rbd_provisioner_enabled: >- - {%- if taco_storage_backend == 'ceph' -%}true{%- else -%}false{%- endif -%} - -# below will be used by default when rbd_provisioner_enabled=true -rbd_provisioner_namespace: kube-system -rbd_provisioner_replicas: 1 -rbd_provisioner_pool: kube -rbd_provisioner_secret_name: ceph-secret-admin -rbd_provisioner_user_secret_name: ceph-secret-user -rbd_provisioner_user_secret_namespace: kube-system -rbd_provisioner_fs_type: ext4 -rbd_provisioner_image_format: "2" -rbd_provisioner_image_features: layering -rbd_provisioner_storage_class: rbd -rbd_provisioner_reclaim_policy: Delete -rbd_provisioner_admin_id: "kube" -rbd_provisioner_secret: "AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ==" -rbd_provisioner_user_id: "kube" -rbd_provisioner_user_secret: "AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ==" - -rbd_provisioner_image_repo: "docker.io/sktdev/rbd-provisioner" -rbd_provisioner_image_tag: "v2.1.1-nautilus-14.2.4" - # external provisioners > local_provisioners # For more customization, refer to "roles/kubernetes-apps/external_provisioner local_volume_provisioner_enabled: false @@ -81,7 +57,6 @@ kube_proxy_metrics_bind_address: 0.0.0.0:10249 #kubelet_deployment_type: host #local_volume_provisioner_enabled: false -#cephfs_provisioner_enabled: false # Applications #dns_mode: coredns diff --git a/defaults/global_k8s-images.yml b/defaults/global_k8s-images.yml index 70d9d45b..97974774 100644 --- a/defaults/global_k8s-images.yml +++ b/defaults/global_k8s-images.yml @@ -4,4 +4,3 @@ quay_image_repo: "{{ container_registry_url }}" kube_image_repo: "{{ container_registry_url }}" armada_image_repo: "{{ container_registry_url }}/sktdev/armada" openstackclient_image_repo: "{{ container_registry_url }}/sktdev/openstackclient" -rbd_provisioner_image_repo: "{{ container_registry_url }}/sktdev/rbd-provisioner" diff --git a/defaults/global_taco.yml b/defaults/global_taco.yml index 70dd23be..f1eb17b9 100644 --- a/defaults/global_taco.yml +++ b/defaults/global_taco.yml @@ -17,17 +17,14 @@ docker_insecure_registries: [] pip_repo_url: "" pkg_repo_url: "" k8s_binary_repo_url: "" -ceph_repo_url: "" -# Storage configurations: ceph or nfs +# Storage configurations: rook-ceph or nfs taco_storage_backend: "rook-ceph" # Taco Apps to deploy using Armada taco_apps: ["openstack"] # These images are managed by TACO -rbd_provisioner_image_repo: "docker.io/sktdev/rbd-provisioner" -rbd_provisioner_image_tag: "v2.1.1-nautilus-14.2.4" openstackclient_image_repo: "docker.io/sktdev/openstackclient" openstackclient_image_tag: "stein" diff --git a/include_defaults.yml b/include_defaults.yml index 0999a3e1..28aef8d2 100644 --- a/include_defaults.yml +++ b/include_defaults.yml @@ -7,12 +7,6 @@ - "{{ playbook_dir }}/defaults/global_taco.yml" - "{{ playbook_dir }}/defaults/global_k8s-cluster.yml" - - name: include global override values - ceph - include_vars: "{{ item }}" - loop: - - "{{ playbook_dir }}/defaults/global_ceph.yml" - when: taco_storage_backend == 'ceph' - - name: include global override values - k8s images include_vars: "{{ item }}" loop: diff --git a/roles/setup-os/conf-repos/defaults/main.yml b/roles/setup-os/conf-repos/defaults/main.yml index e8a12106..a984e225 100644 --- a/roles/setup-os/conf-repos/defaults/main.yml +++ b/roles/setup-os/conf-repos/defaults/main.yml @@ -2,4 +2,3 @@ pip_repo_url: "" pkg_repo_url: "" k8s_binary_repo_url: "" -ceph_repo_url: "" diff --git a/roles/setup-os/conf-repos/tasks/main.yml b/roles/setup-os/conf-repos/tasks/main.yml index 3db92c9c..c748e98d 100644 --- a/roles/setup-os/conf-repos/tasks/main.yml +++ b/roles/setup-os/conf-repos/tasks/main.yml @@ -42,30 +42,6 @@ - pkg_repo_url != "" - ansible_distribution in ["Ubuntu","Debian"] -- name: set ceph_custom_repo for ceph-ansible - set_fact: - ceph_custom_repo: "http://{{ ceph_repo_url }}/ceph/ceph.repo" - when: - - ansible_distribution in ["CentOS", "RedHat"] - - ceph_repo_url != "" - -- name: set ceph_custom_repo for ceph-ansible - set_fact: - ceph_custom_repo: "http://{{ ceph_repo_url }}/ceph/debian-nautilus" - when: - - ceph_repo_url != "" - - ansible_distribution in ["Ubuntu", "Debian"] - -- name: configure debian/ubuntu ceph stable repository key - apt_key: - data: "{{ lookup('file', playbook_dir+'/ceph-ansible/roles/ceph-common/files/cephstable.asc') }}" - state: present - register: result - until: result is succeeded - when: - - ceph_repo_url != "" - - ansible_distribution in ["Ubuntu", "Debian"] - - name: add local pypi repo template: src: pip.conf.j2 From 775200afc864b7272eb8106ae1bd50134749e71a Mon Sep 17 00:00:00 2001 From: Jugwan Eom Date: Thu, 6 May 2021 10:45:21 +0000 Subject: [PATCH 4/4] inventory/sample/aio: use rook-ceph --- inventory/sample/aio/extra-vars.yml | 72 ++--------------------------- 1 file changed, 4 insertions(+), 68 deletions(-) diff --git a/inventory/sample/aio/extra-vars.yml b/inventory/sample/aio/extra-vars.yml index 83cc8542..3c06b526 100644 --- a/inventory/sample/aio/extra-vars.yml +++ b/inventory/sample/aio/extra-vars.yml @@ -1,7 +1,9 @@ # tacoplay -# TACO parameter : a backend storage for kubernetes shared storage (ceph|nfs) -taco_storage_backend: "ceph" +# ceph +rook_ceph_cluster_mon_replicas: 1 +rook_ceph_cluster_taco_pool_size: 1 +rook_ceph_cluster_taco_pool_require_safe_size: "false" # TACO parameter : list of applications that will be deployed on kubernetes (openstack|lma|etc) taco_apps: [] @@ -9,21 +11,6 @@ taco_apps: [] # TACO parameter : container image registry will be deployed on the node defined in container-registry in hosts.ini container_registry_enabled: true -# ceph parameter -monitor_interface: br-data -public_network: 192.168.97.0/24 -cluster_network: 192.168.97.0/24 - -ceph_conf_overrides: - global: - mon_allow_pool_delete: true - osd_pool_default_size: 1 - osd_pool_default_min_size: 1 - -osd_objectstore: bluestore -lvm_volumes: - - data: /dev/vdb - # kubespray parameter : enable calico IP-in-IP encapsulation of the inter-workload traffic. (Always | Never | CrossSubnet) ipip_mode: Always #peer_with_router: false @@ -31,57 +18,6 @@ ipip_mode: Always # Override openstack release (Default value is defined in Tacoplay) #Openstack_release: stein -# OpenStack parameter -openstack_config: true -kube_pool: - name: "kube" - pg_num: 2 - pgp_num: 2 - rule_name: "replicated_rule" - type: 1 - erasure_profile: "" - expected_num_objects: "" - application: "rbd" -openstack_glance_pool: - name: "images" - pg_num: 8 - pgp_num: 8 - rule_name: "replicated_rule" - type: 1 - erasure_profile: "" - expected_num_objects: "" -openstack_cinder_pool: - name: "volumes" - pg_num: 16 - pgp_num: 16 - rule_name: "replicated_rule" - type: 1 - erasure_profile: "" - expected_num_objects: "" -openstack_cinder_backup_pool: - name: "backups" - pg_num: 8 - pgp_num: 8 - rule_name: "replicated_rule" - type: 1 - erasure_profile: "" - expected_num_objects: "" -openstack_nova_vms_pool: - name: "vms" - pg_num: 16 - pgp_num: 16 - rule_name: "replicated_rule" - type: 1 - erasure_profile: "" - expected_num_objects: "" - -openstack_pools: - - "{{ kube_pool }}" - - "{{ openstack_glance_pool }}" - - "{{ openstack_cinder_pool }}" - - "{{ openstack_cinder_backup_pool }}" - - "{{ openstack_nova_vms_pool }}" - # OpenStack ACCOUNT os_root_user: admin os_root_password: password