Merge pull request #72 from openinfradev/rook_ceph

Rook ceph 도입
openinfradev · May 7, 2021 · e1e04aa · e1e04aa
2 parents 558a877 + 775200a
commit e1e04aa
Show file tree

Hide file tree

Showing 16 changed files with 66 additions and 213 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,7 +4,6 @@
 artifacts/
 docker_registry/
 /armada/
-ceph-ansible/
 kubespray/
 charts/
 mirrors/

diff --git a/VERSIONS b/VERSIONS
@@ -1,6 +1,5 @@
 kubespray https://github.com/openinfradev/kubespray.git v2.15.1
 #charts/openstack-helm https://github.com/openinfradev/openstack-helm.git master
 #charts/openstack-helm-infra https://github.com/openinfradev/openstack-helm-infra.git master
-ceph-ansible https://github.com/openinfradev/ceph-ansible.git stable-4.0
 charts/taco-helm-charts https://github.com/openinfradev/helm-charts.git main
 charts/argo-helm https://github.com/argoproj/argo-helm.git master
diff --git a/defaults/TACOFIXME.yml b/defaults/TACOFIXME.yml
@@ -3,6 +3,7 @@
 
 # TACO
 
+
 # Ceph
 # ceph_monitors and ceph_admin_keyring would be set automatically
 # from /etc/ceph/ceph.conf and ceph.client.admin.keyring
@@ -15,12 +16,7 @@ ceph_admin_keyring: >-
   {%- if groups["osds"] is defined -%}{%- else -%}TACOFIXME{%- endif -%}
 
 taco_storageclass_name: >-
-  {%- if taco_storage_backend == 'ceph' -%}rbd{%- elif taco_storage_backend == 'nfs' -%}nfs-client{%- else -%}TACOFIXME{%- endif -%}
-
-# when deploying new ceph
-monitor_interface: TACOFIXME
-public_network: TACOFIXME
-cluster_network: TACOFIXME
+  {%- if taco_storage_backend == 'rook-ceph' -%}rbd{%- elif taco_storage_backend == 'nfs' -%}nfs-client{%- else -%}TACOFIXME{%- endif -%}
 
 # K8S
 kube_pods_subnet: TACOFIXME
@@ -36,17 +32,6 @@ ipip_mode: TACOFIXME
 # as number and border router IP should be set only if it is enabled
 peer_with_router: TACOFIXME
 
-# RBD provisioner
-# set below only when using existing ceph
-rbd_provisioner_admin_id: >-
-  {%- if groups["osds"] is not defined -%}TACOFIXME{%- else -%}kube{%- endif -%}
-rbd_provisioner_secret: >-
-  {%- if groups["osds"] is not defined -%}TACOFIXME{%- else -%}AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ=={%- endif -%}
-rbd_provisioner_user_id: >-
-  {%- if groups["osds"] is not defined -%}TACOFIXME{%- else -%}kube{%- endif -%}
-rbd_provisioner_user_secret: >-
-  {%- if groups["osds"] is not defined -%}TACOFIXME{%- else -%}AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ=={%- endif -%}
-
 # external NFS server
 nfs_server: TACOFIXME
 nfs_path: TACOFIXME

diff --git a/defaults/global_ceph.yml b/defaults/global_ceph.yml
diff --git a/defaults/global_k8s-cluster.yml b/defaults/global_k8s-cluster.yml
@@ -17,30 +17,6 @@ calico_felix_prometheusmetricsport: 9091
 calico_felix_prometheusgometricsenabled: "true"
 calico_felix_prometheusprocessmetricsenabled: "true"
 
-# external provisioners > rbd_provisioner
-rbd_provisioner_enabled: >-
-    {%- if taco_storage_backend == 'ceph' -%}true{%- else -%}false{%- endif -%}
-
-# below will be used by default when rbd_provisioner_enabled=true
-rbd_provisioner_namespace: kube-system
-rbd_provisioner_replicas: 1
-rbd_provisioner_pool: kube
-rbd_provisioner_secret_name: ceph-secret-admin
-rbd_provisioner_user_secret_name: ceph-secret-user
-rbd_provisioner_user_secret_namespace: kube-system
-rbd_provisioner_fs_type: ext4
-rbd_provisioner_image_format: "2"
-rbd_provisioner_image_features: layering
-rbd_provisioner_storage_class: rbd
-rbd_provisioner_reclaim_policy: Delete
-rbd_provisioner_admin_id: "kube"
-rbd_provisioner_secret: "AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ=="
-rbd_provisioner_user_id: "kube"
-rbd_provisioner_user_secret: "AQAPn8tUmPBwCxAAeIfvpDKA1fGvrBeXGdc6xQ=="
-
-rbd_provisioner_image_repo: "docker.io/sktdev/rbd-provisioner"
-rbd_provisioner_image_tag: "v2.1.1-nautilus-14.2.4"
-
 # external provisioners > local_provisioners
 # For more customization, refer to "roles/kubernetes-apps/external_provisioner
 local_volume_provisioner_enabled: false
@@ -81,7 +57,6 @@ kube_proxy_metrics_bind_address: 0.0.0.0:10249
 #kubelet_deployment_type: host
 
 #local_volume_provisioner_enabled: false
-#cephfs_provisioner_enabled: false
 
 # Applications
 #dns_mode: coredns

diff --git a/defaults/global_k8s-images.yml b/defaults/global_k8s-images.yml
@@ -4,4 +4,3 @@ quay_image_repo: "{{ container_registry_url }}"
 kube_image_repo: "{{ container_registry_url }}"
 armada_image_repo: "{{ container_registry_url }}/sktdev/armada"
 openstackclient_image_repo: "{{ container_registry_url }}/sktdev/openstackclient"
-rbd_provisioner_image_repo: "{{ container_registry_url }}/sktdev/rbd-provisioner"
diff --git a/defaults/global_taco.yml b/defaults/global_taco.yml
@@ -17,17 +17,14 @@ docker_insecure_registries: []
 pip_repo_url: ""
 pkg_repo_url: ""
 k8s_binary_repo_url: ""
-ceph_repo_url: ""
 
-# Storage configurations: ceph or nfs
-taco_storage_backend: "ceph"
+# Storage configurations: rook-ceph or nfs
+taco_storage_backend: "rook-ceph"
 
 # Taco Apps to deploy using Armada
 taco_apps: ["openstack"]
 
 # These images are managed by TACO
-rbd_provisioner_image_repo: "docker.io/sktdev/rbd-provisioner"
-rbd_provisioner_image_tag: "v2.1.1-nautilus-14.2.4"
 openstackclient_image_repo: "docker.io/sktdev/openstackclient"
 openstackclient_image_tag: "stein"
 

diff --git a/include_defaults.yml b/include_defaults.yml
@@ -7,12 +7,6 @@
         - "{{ playbook_dir }}/defaults/global_taco.yml"
         - "{{ playbook_dir }}/defaults/global_k8s-cluster.yml"
 
-    - name: include global override values - ceph
-      include_vars: "{{ item }}"
-      loop:
-        - "{{ playbook_dir }}/defaults/global_ceph.yml"
-      when: taco_storage_backend == 'ceph'
-
     - name: include global override values - k8s images
       include_vars: "{{ item }}"
       loop:

diff --git a/inventory/sample/aio/extra-vars.yml b/inventory/sample/aio/extra-vars.yml
@@ -1,87 +1,23 @@
 # tacoplay
 
-# TACO parameter : a backend storage for kubernetes shared storage (ceph|nfs)
-taco_storage_backend: "ceph"
+# ceph
+rook_ceph_cluster_mon_replicas: 1
+rook_ceph_cluster_taco_pool_size: 1
+rook_ceph_cluster_taco_pool_require_safe_size: "false"
 
 # TACO parameter : list of applications that will be deployed on kubernetes (openstack|lma|etc)
 taco_apps: []
 
 # TACO parameter : container image registry will be deployed on the node defined in container-registry in hosts.ini
 container_registry_enabled: true
 
-# ceph parameter
-monitor_interface: br-data
-public_network: 192.168.97.0/24
-cluster_network: 192.168.97.0/24
-
-ceph_conf_overrides:
-  global:
-    mon_allow_pool_delete: true
-    osd_pool_default_size: 1
-    osd_pool_default_min_size: 1
-
-osd_objectstore: bluestore
-lvm_volumes:
-  - data: /dev/vdb
-
 # kubespray parameter : enable calico IP-in-IP encapsulation of the inter-workload traffic. (Always | Never | CrossSubnet)
 ipip_mode: Always
 #peer_with_router: false
 
 # Override openstack release (Default value is defined in Tacoplay)
 #Openstack_release: stein
 
-# OpenStack parameter
-openstack_config: true
-kube_pool:
-  name: "kube"
-  pg_num: 2
-  pgp_num: 2
-  rule_name: "replicated_rule"
-  type: 1
-  erasure_profile: ""
-  expected_num_objects: ""
-  application: "rbd"
-openstack_glance_pool:
-  name: "images"
-  pg_num: 8
-  pgp_num: 8
-  rule_name: "replicated_rule"
-  type: 1
-  erasure_profile: ""
-  expected_num_objects: ""
-openstack_cinder_pool:
-  name: "volumes"
-  pg_num: 16
-  pgp_num: 16
-  rule_name: "replicated_rule"
-  type: 1
-  erasure_profile: ""
-  expected_num_objects: ""
-openstack_cinder_backup_pool:
-  name: "backups"
-  pg_num: 8
-  pgp_num: 8
-  rule_name: "replicated_rule"
-  type: 1
-  erasure_profile: ""
-  expected_num_objects: ""
-openstack_nova_vms_pool:
-  name: "vms"
-  pg_num: 16
-  pgp_num: 16
-  rule_name: "replicated_rule"
-  type: 1
-  erasure_profile: ""
-  expected_num_objects: ""
-
-openstack_pools:
-  - "{{ kube_pool }}"
-  - "{{ openstack_glance_pool }}"
-  - "{{ openstack_cinder_pool }}"
-  - "{{ openstack_cinder_backup_pool }}"
-  - "{{ openstack_nova_vms_pool }}"
-
 # OpenStack ACCOUNT
 os_root_user: admin
 os_root_password: password

diff --git a/requirements.txt b/requirements.txt
@@ -1,12 +1,11 @@
-# from kubespray v2.13.1
-ansible==2.10.0
+# from kubespray v2.15.1
+ansible==2.9.20
+cryptography==2.8
 jinja2==2.11.3
 netaddr==0.7.19
 pbr==5.4.4
-hvac==0.10.0
 jmespath==0.9.5
 ruamel.yaml==0.16.10
 
 # for taco
 docker
-cryptography
diff --git a/reset.yml b/reset.yml
@@ -10,8 +10,3 @@
 # Reset K8S cluster
 - include: kubespray/reset.yml
   tags: k8s
-
-# Purge Ceph cluster
-- include: ceph-ansible/infrastructure-playbooks/purge-cluster.yml
-  tags: ceph
-  when: taco_storage_backend == 'ceph'
diff --git a/roles/ceph/rook/defaults/main.yml b/roles/ceph/rook/defaults/main.yml
@@ -0,0 +1,5 @@
+---
+rook_ceph_cluster_chart_source: "{{ lookup('env', 'HOME') }}/tacoplay/charts/taco-helm-charts/rook-ceph-cluster"
+rook_ceph_cluster_mon_replicas: 3
+rook_ceph_cluster_taco_pool_size: 3
+rook_ceph_cluster_taco_pool_require_safe_size: "true"
diff --git a/roles/ceph/rook/tasks/main.yml b/roles/ceph/rook/tasks/main.yml
@@ -0,0 +1,49 @@
+---
+- name: create namespace for rook
+  shell: >-
+    {{ bin_dir }}/kubectl create namespace rook-ceph
+  ignore_errors: true
+  become: false
+
+- name: add rook release helm repository
+  shell: >-
+    {{ bin_dir }}/helm repo add rook-release https://charts.rook.io/release
+  become: false
+
+- name: install rook-operator chart
+  shell: >-
+    {{ bin_dir }}/helm install --namespace rook-ceph rook-ceph rook-release/rook-ceph
+  become: false
+
+- name: sleep for 60 seconds for rook-operator pod to be launched
+  wait_for:
+    timeout: 60
+
+- name: wait for rook-operator pods become ready
+  shell: >-
+    {{ bin_dir }}/kubectl wait --namespace=rook-ceph --for=condition=Ready pods -l app={{ item }} --timeout=600s
+  become: false
+  delay: 10
+  retries: 3
+  with_items:
+    - rook-ceph-operator
+
+- name: install rook ceph cluster chart
+  shell: >-
+    {{ bin_dir }}/helm install --namespace rook-ceph rook-ceph-cluster {{ rook_ceph_cluster_chart_source }} \
+    --set cluster.mon.count={{ rook_ceph_cluster_mon_replicas }} \
+    --set block_pools[0].name=taco \
+    --set block_pools[0].size={{ rook_ceph_cluster_taco_pool_size  }} \
+    --set block_pools[0].requireSafeReplicaSize={{ rook_ceph_cluster_taco_pool_require_safe_size }}
+  become: false
+
+- name: sleep for 300 seconds for rook ceph cluster to be initialized
+  wait_for:
+    timeout: 300
+
+- name: wait for rook ceph cluster become ready
+  shell: >-
+    {{ bin_dir }}/kubectl wait -n rook-ceph --for=condition=Ready cephcluster rook-ceph --timeout=600s
+  become: false
+  delay: 10
+  retries: 3
diff --git a/roles/setup-os/conf-repos/defaults/main.yml b/roles/setup-os/conf-repos/defaults/main.yml
@@ -2,4 +2,3 @@
 pip_repo_url: ""
 pkg_repo_url: ""
 k8s_binary_repo_url: ""
-ceph_repo_url: ""
diff --git a/roles/setup-os/conf-repos/tasks/main.yml b/roles/setup-os/conf-repos/tasks/main.yml
@@ -42,30 +42,6 @@
     - pkg_repo_url != ""
     - ansible_distribution in ["Ubuntu","Debian"]
 
-- name: set ceph_custom_repo for ceph-ansible
-  set_fact:
-    ceph_custom_repo: "http://{{ ceph_repo_url }}/ceph/ceph.repo"
-  when:
-    - ansible_distribution in ["CentOS", "RedHat"]
-    - ceph_repo_url != ""
-
-- name: set ceph_custom_repo for ceph-ansible
-  set_fact:
-    ceph_custom_repo: "http://{{ ceph_repo_url }}/ceph/debian-nautilus"
-  when:
-    - ceph_repo_url != ""
-    - ansible_distribution in ["Ubuntu", "Debian"]
-
-- name: configure debian/ubuntu ceph stable repository key
-  apt_key:
-    data: "{{ lookup('file', playbook_dir+'/ceph-ansible/roles/ceph-common/files/cephstable.asc') }}"
-    state: present
-  register: result
-  until: result is succeeded
-  when:
-    - ceph_repo_url != ""
-    - ansible_distribution in ["Ubuntu", "Debian"]
-
 - name: add local pypi repo
   template:
     src: pip.conf.j2