From c38f43d7e3d6ce5bedd3ff17e61978200656d78b Mon Sep 17 00:00:00 2001 From: fightdou <1426589337@qq.com> Date: Wed, 13 Jul 2022 10:35:36 +0800 Subject: [PATCH] [cinder] Add some features and fix some bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1、Map log file to host directory 2、Auto create volume type 3、Fix some typos. 4、Add related privileges about ceph.rook.io and cephclients to cinder serviceaccount. --- charts/cinder/Chart.yaml | 2 +- charts/cinder/templates/bin/_lvm_clean.sh.tpl | 2 +- .../templates/bin/_volume_type_create.sh.tpl | 11 +++ .../cinder/templates/cephclient-cinder.yaml | 2 +- charts/cinder/templates/cephpool-backup.yaml | 2 +- charts/cinder/templates/cephpool-volumes.yaml | 2 +- charts/cinder/templates/cinder-rbac.yaml | 20 ++-- charts/cinder/templates/conf/_cinder.conf.tpl | 4 +- charts/cinder/templates/configmap-bin.yaml | 2 + charts/cinder/templates/daemonset-volume.yaml | 41 ++++---- charts/cinder/templates/deployment-api.yaml | 10 +- .../cinder/templates/deployment-backup.yaml | 21 +++- .../templates/deployment-scheduler.yaml | 10 +- .../templates/job-volume-type-create.yaml | 95 +++++++++++++++++++ 14 files changed, 186 insertions(+), 38 deletions(-) create mode 100644 charts/cinder/templates/bin/_volume_type_create.sh.tpl create mode 100644 charts/cinder/templates/job-volume-type-create.yaml diff --git a/charts/cinder/Chart.yaml b/charts/cinder/Chart.yaml index 2c32052..0f6bb5f 100644 --- a/charts/cinder/Chart.yaml +++ b/charts/cinder/Chart.yaml @@ -13,7 +13,7 @@ apiVersion: v2 description: Openstack cinder service name: cinder -version: 1.0.1 +version: 1.0.2 home: https://github.com/kungze/kolla-helm maintainers: - name: Kungze diff --git a/charts/cinder/templates/bin/_lvm_clean.sh.tpl b/charts/cinder/templates/bin/_lvm_clean.sh.tpl index 081f578..83e5354 100644 --- a/charts/cinder/templates/bin/_lvm_clean.sh.tpl +++ b/charts/cinder/templates/bin/_lvm_clean.sh.tpl @@ -1,7 +1,7 @@ #!/bin/bash set -ex vgremove -y {{ .Values.lvm.vg_name }} -pvremove {{ .Values.lvm.loop_device_name }} +pvremove {{ .Values.lvm.loop_device_name }} --force --force {{- if .Values.lvm.create_loop_device }} losetup -d {{ .Values.lvm.loop_device_name }} {{- end }} diff --git a/charts/cinder/templates/bin/_volume_type_create.sh.tpl b/charts/cinder/templates/bin/_volume_type_create.sh.tpl new file mode 100644 index 0000000..a318234 --- /dev/null +++ b/charts/cinder/templates/bin/_volume_type_create.sh.tpl @@ -0,0 +1,11 @@ +#!/bin/bash +set -ex +{{- if .Values.ceph.enabled }} +openstack volume type create {{ .Values.ceph.volume_type }} +openstack volume type set --property volume_backend_name='{{ .Values.ceph.volume_type }}' {{ .Values.ceph.volume_type }} +{{- end }} + +{{- if .Values.lvm.enabled }} +openstack volume type create {{ .Values.lvm.volume_type }} +openstack volume type set --property volume_backend_name='{{ .Values.lvm.volume_type }}' {{ .Values.lvm.volume_type }} +{{- end }} diff --git a/charts/cinder/templates/cephclient-cinder.yaml b/charts/cinder/templates/cephclient-cinder.yaml index 2496340..5af9b5c 100644 --- a/charts/cinder/templates/cephclient-cinder.yaml +++ b/charts/cinder/templates/cephclient-cinder.yaml @@ -11,4 +11,4 @@ {{ $cephClient := dict "cephClusterNamespace" .Values.ceph.cephClusterNamespace "cephUserName" .Values.ceph.cephClientName "prvileges" $prvileges }} {{- $cephClient | include "common.manifests.cephclient" -}} {{- end -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/cinder/templates/cephpool-backup.yaml b/charts/cinder/templates/cephpool-backup.yaml index 25c3501..bb7ea91 100644 --- a/charts/cinder/templates/cephpool-backup.yaml +++ b/charts/cinder/templates/cephpool-backup.yaml @@ -1,4 +1,4 @@ {{- if and .Values.ceph.enabled .Values.ceph.backup.enabled -}} {{- $cephPool := dict "poolName" .Values.ceph.backup.poolName "cephClusterNamespace" .Values.ceph.cephClusterNamespace "replicated" .Values.ceph.backup.replicatedSize "failureDomain" .Values.ceph.backup.failureDomain -}} {{- $cephPool | include "common.manifests.cephpool" -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/cinder/templates/cephpool-volumes.yaml b/charts/cinder/templates/cephpool-volumes.yaml index dd95c59..e8853b6 100644 --- a/charts/cinder/templates/cephpool-volumes.yaml +++ b/charts/cinder/templates/cephpool-volumes.yaml @@ -1,4 +1,4 @@ {{- if .Values.ceph.enabled -}} {{- $cephPool := dict "poolName" .Values.ceph.poolName "cephClusterNamespace" .Values.ceph.cephClusterNamespace "replicated" .Values.ceph.replicatedSize "failureDomain" .Values.ceph.failureDomain -}} {{- $cephPool | include "common.manifests.cephpool" -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/cinder/templates/cinder-rbac.yaml b/charts/cinder/templates/cinder-rbac.yaml index 3df5c06..09bf7b8 100644 --- a/charts/cinder/templates/cinder-rbac.yaml +++ b/charts/cinder/templates/cinder-rbac.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} @@ -9,7 +9,7 @@ metadata: apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} rules: @@ -35,16 +35,16 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace | quote }} {{- if .Values.ceph.enabled }} @@ -52,7 +52,7 @@ subjects: kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} rules: @@ -60,9 +60,11 @@ rules: - "" - apps - extensions + - ceph.rook.io resources: - secrets - configmaps + - cephclients verbs: - get - list @@ -76,15 +78,15 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace | quote }} labels: {{- include "common.labels.standard" . | nindent 4 }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccountName}} + name: {{ .Values.serviceAccountName }} namespace: {{ .Release.Namespace | quote }} {{- end }} diff --git a/charts/cinder/templates/conf/_cinder.conf.tpl b/charts/cinder/templates/conf/_cinder.conf.tpl index f663445..c4742e2 100644 --- a/charts/cinder/templates/conf/_cinder.conf.tpl +++ b/charts/cinder/templates/conf/_cinder.conf.tpl @@ -2,6 +2,7 @@ debug = False use_forwarded_for = true use_stderr = False +log_dir = /var/log/kolla/cinder osapi_volume_workers = 5 volume_name_template = volume-%s volumes_dir = /var/lib/cinder/volumes @@ -18,7 +19,7 @@ enabled_backends = {{ .Values.ceph.volume_type }} default_volume_type = {{ .Values.ceph.volume_type }} {{- end }} -{{- if .Values.ceph.backup.anabled }} +{{- if and .Values.ceph.enabled .Values.ceph.backup.enabled }} backup_driver = cinder.backup.drivers.ceph.CephBackupDriver backup_ceph_conf = /etc/ceph/ceph.conf backup_ceph_user = {{ .Values.ceph.cephClientName }} @@ -99,4 +100,5 @@ volume_driver = cinder.volume.drivers.lvm.LVMVolumeDriver volume_backend_name = {{ .Values.lvm.volume_type }} target_helper = {{ .Values.lvm.lvm_target_helper }} target_protocol = iscsi +lvm_type = default {{- end }} diff --git a/charts/cinder/templates/configmap-bin.yaml b/charts/cinder/templates/configmap-bin.yaml index 4f19c4b..b65b550 100644 --- a/charts/cinder/templates/configmap-bin.yaml +++ b/charts/cinder/templates/configmap-bin.yaml @@ -28,3 +28,5 @@ data: sync-ceph-cm-secrets.py: | {{- include "common.scripts.sync_ceph_cm_secret" . | indent 4 }} {{- end }} + volume-type-create.sh: | +{{ tuple "bin/_volume_type_create.sh.tpl" . | include "common.utils.template" | indent 4 }} diff --git a/charts/cinder/templates/daemonset-volume.yaml b/charts/cinder/templates/daemonset-volume.yaml index a7ec98d..74b5f50 100644 --- a/charts/cinder/templates/daemonset-volume.yaml +++ b/charts/cinder/templates/daemonset-volume.yaml @@ -13,6 +13,7 @@ spec: app.kubernetes.io/component: cinder-volume spec: hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet initContainers: - name: init command: @@ -34,7 +35,7 @@ spec: value: eth0 - name: DEPENDENCY_SERVICE value: "cinder-api" - {{- if .Values.lvm.create_loop_device }} + {{- if and .Values.lvm.enabled .Values.lvm.create_loop_device }} - name: loop image: {{ template "cinder.loop.image" . }} imagePullPolicy: {{ .Values.pullPolicy }} @@ -106,6 +107,8 @@ spec: key: {{ .Values.ceph.cephClientName }} {{- end }} volumeMounts: + - mountPath: /var/log/kolla/cinder + name: cinderlog - mountPath: /tmp name: pod-tmp - mountPath: /dev @@ -129,7 +132,6 @@ spec: - mountPath: /tmp/lvm-clean.sh name: cinder-bin subPath: lvm-clean.sh - {{- end }} {{- if eq .Values.lvm.lvm_target_helper "tgtadm" }} - mountPath: /var/lib/cinder/volumes name: cinderlib @@ -140,6 +142,7 @@ spec: mountPath: /etc/multipath mountPropagation: Bidirectional {{- end }} + {{- end }} {{- if .Values.ceph.enabled }} - mountPath: /etc/ceph name: etc-ceph @@ -149,7 +152,7 @@ spec: - mountPath: /etc/rook name: mon-endpoint-volume {{- end }} - {{- if eq .Values.lvm.lvm_target_helper "tgtadm" }} + {{- if and .Values.lvm.enabled (eq .Values.lvm.lvm_target_helper "tgtadm") }} - name: tgtd image: {{ template "kolla.tgtd.image" . }} imagePullPolicy: {{ .Values.pullPolicy }} @@ -182,17 +185,31 @@ spec: - mountPath: /var/lib/kolla/config_files/config.json name: cinder-etc subPath: tgtd.json - {{- end }} - serviceAccount: {{ .Values.serviceAccountName}} - serviceAccountName: {{ .Values.serviceAccountName}} + {{- end }} + serviceAccount: {{ .Values.serviceAccountName }} + serviceAccountName: {{ .Values.serviceAccountName }} volumes: + - hostPath: + path: /var/log/kolla/cinder + type: DirectoryOrCreate + name: cinderlog - emptyDir: {} name: pod-tmp - {{- if and .Values.lvm.enabled }} + {{- if .Values.lvm.create_loop_device }} - name: loop-device-dir hostPath: path: {{ .Values.lvm.loop_device_directory | quote }} type: DirectoryOrCreate + {{- end }} + {{- if and .Values.lvm.enabled (eq .Values.lvm.lvm_target_helper "tgtadm") }} + - emptyDir: {} + name: cinderlib + - name: etciscsi + hostPath: + path: /etc/iscsi + - name: etcmultipath + hostPath: + path: /etc/multipath {{- end -}} {{- if .Values.ceph.enabled }} - emptyDir: {} @@ -228,13 +245,3 @@ spec: hostPath: path: /sys type: DirectoryOrCreate - {{- if eq .Values.lvm.lvm_target_helper "tgtadm" }} - - emptyDir: {} - name: cinderlib - - name: etciscsi - hostPath: - path: /etc/iscsi - - name: etcmultipath - hostPath: - path: /etc/multipath - {{- end }} diff --git a/charts/cinder/templates/deployment-api.yaml b/charts/cinder/templates/deployment-api.yaml index d888950..9383ddf 100644 --- a/charts/cinder/templates/deployment-api.yaml +++ b/charts/cinder/templates/deployment-api.yaml @@ -70,6 +70,8 @@ spec: containerPort: 8776 protocol: TCP volumeMounts: + - mountPath: /var/log/kolla/cinder + name: cinderlog - mountPath: /tmp name: pod-tmp - mountPath: /var/lib/kolla/config_files/cinder.conf @@ -81,9 +83,13 @@ spec: - mountPath: /var/lib/kolla/config_files/cinder-wsgi.conf name: cinder-etc subPath: cinder-wsgi.conf - serviceAccount: {{ .Values.serviceAccountName}} - serviceAccountName: {{ .Values.serviceAccountName}} + serviceAccount: {{ .Values.serviceAccountName }} + serviceAccountName: {{ .Values.serviceAccountName }} volumes: + - hostPath: + path: /var/log/kolla/cinder + type: DirectoryOrCreate + name: cinderlog - emptyDir: {} name: pod-tmp - configMap: diff --git a/charts/cinder/templates/deployment-backup.yaml b/charts/cinder/templates/deployment-backup.yaml index 9238c0e..e5d30df 100644 --- a/charts/cinder/templates/deployment-backup.yaml +++ b/charts/cinder/templates/deployment-backup.yaml @@ -46,12 +46,25 @@ spec: exec: command: - /tmp/gen-ceph-conf.sh + securityContext: + privileged: true env: - name: KOLLA_CONFIG_STRATEGY value: "COPY_ALWAYS" - name: KOLLA_SERVICE_NAME value: "cinder-backup" + {{- if .Values.ceph.enabled }} + - name: ROOK_CEPH_USERNAME + value: {{ .Values.ceph.cephClientName | quote }} + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + name: {{ printf "ceph-%s" .Values.ceph.cephClientName | quote }} + key: {{ .Values.ceph.cephClientName }} + {{- end }} volumeMounts: + - mountPath: /var/log/kolla/cinder + name: cinderlog - mountPath: /tmp name: pod-tmp - mountPath: /var/lib/kolla/config_files/cinder.conf @@ -67,9 +80,13 @@ spec: subPath: gen-ceph-conf.sh - mountPath: /etc/rook name: mon-endpoint-volume - serviceAccount: {{ .Values.serviceAccountName}} - serviceAccountName: {{ .Values.serviceAccountName}} + serviceAccount: {{ .Values.serviceAccountName }} + serviceAccountName: {{ .Values.serviceAccountName }} volumes: + - hostPath: + path: /var/log/kolla/cinder + type: DirectoryOrCreate + name: cinderlog - emptyDir: {} name: pod-tmp - emptyDir: {} diff --git a/charts/cinder/templates/deployment-scheduler.yaml b/charts/cinder/templates/deployment-scheduler.yaml index 7c0f988..5c9420a 100644 --- a/charts/cinder/templates/deployment-scheduler.yaml +++ b/charts/cinder/templates/deployment-scheduler.yaml @@ -46,6 +46,8 @@ spec: - name: KOLLA_SERVICE_NAME value: "cinder-scheduler" volumeMounts: + - mountPath: /var/log/kolla/cinder + name: cinderlog - mountPath: /tmp name: pod-tmp - mountPath: /var/lib/kolla/config_files/cinder.conf @@ -54,9 +56,13 @@ spec: - mountPath: /var/lib/kolla/config_files/config.json name: cinder-etc subPath: cinder-scheduler.json - serviceAccount: {{ .Values.serviceAccountName}} - serviceAccountName: {{ .Values.serviceAccountName}} + serviceAccount: {{ .Values.serviceAccountName }} + serviceAccountName: {{ .Values.serviceAccountName }} volumes: + - hostPath: + path: /var/log/kolla/cinder + type: DirectoryOrCreate + name: cinderlog - emptyDir: {} name: pod-tmp - configMap: diff --git a/charts/cinder/templates/job-volume-type-create.yaml b/charts/cinder/templates/job-volume-type-create.yaml new file mode 100644 index 0000000..c5de95e --- /dev/null +++ b/charts/cinder/templates/job-volume-type-create.yaml @@ -0,0 +1,95 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: cinder-type-create + namespace: {{ .Release.Namespace | quote }} + labels: {{- include "common.labels.standard" . | nindent 4 }} +spec: + template: + spec: + containers: + - name: cinder-bootstrap + image: {{ template "cinder.api.image" . }} + imagePullPolicy: {{ .Values.pullPolicy }} + command: + - /bin/sh + - -c + - /tmp/volume-type-create.sh + env: + - name: OS_USERNAME + valueFrom: + secretKeyRef: + key: OS_USERNAME + name: {{ .Values.keystoneRelease }} + - name: OS_PASSWORD + valueFrom: + secretKeyRef: + key: keystone-admin-password + name: {{ .Values.passwordRelease }} + - name: OS_AUTH_URL + valueFrom: + secretKeyRef: + key: OS_CLUSTER_URL + name: {{ .Values.keystoneRelease }} + - name: OS_REGION_NAME + valueFrom: + secretKeyRef: + key: OS_REGION_NAME + name: {{ .Values.keystoneRelease }} + - name: OS_PROJECT_DOMAIN_NAME + valueFrom: + secretKeyRef: + key: OS_PROJECT_DOMAIN_NAME + name: {{ .Values.keystoneRelease }} + - name: OS_USER_DOMAIN_NAME + valueFrom: + secretKeyRef: + key: OS_USER_DOMAIN_NAME + name: {{ .Values.keystoneRelease }} + - name: OS_PROJECT_NAME + valueFrom: + secretKeyRef: + key: OS_PROJECT_NAME + name: {{ .Values.keystoneRelease }} + - name: OS_DEFAULT_DOMAIN + valueFrom: + secretKeyRef: + key: OS_DEFAULT_DOMAIN + name: {{ .Values.keystoneRelease }} + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: /tmp/volume-type-create.sh + name: cinder-bin + subPath: volume-type-create.sh + initContainers: + - name: init + image: {{ include "common.images.kubernetes-entrypoint" (dict "registry" .Values.imageRegistry "namespace" .Values.imageNamespace) | quote }} + imagePullPolicy: {{ .Values.pullPolicy }} + command: + - kubernetes-entrypoint + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ + - name: DEPENDENCY_SERVICE + value: "cinder-api" + restartPolicy: OnFailure + serviceAccount: {{ .Values.serviceAccountName }} + serviceAccountName: {{ .Values.serviceAccountName }} + volumes: + - emptyDir: {} + name: pod-tmp + - name: cinder-bin + configMap: + defaultMode: 0755 + name: cinder-bin