Docs & Hack: extraVol changes

This patch makes changes to the getting started scripts and manifest to
adapt to the new extraVol changes.

.gitignore

@@ -34,3 +34,7 @@ CI_TOOLS_REPO
 # generated workspace file
 go.work
 go.work.sum
+
+# Hack files that are modified by scripts
+hack/dev/openstack-ceph.yaml
+hack/dev/openstack-lvm-ceph.yaml

hack/dev/create-ceph.sh

@@ -1,14 +1,44 @@
 #!/bin/env bash
+set -e
 LOCATION=$(realpath `dirname -- $BASH_SOURCE[0]`)
-sudo cp -R "${LOCATION}/ceph" /etc
+SECRET_NAME=${1:-ceph-client-files}
+OPENSTACK_YAMLS="$LOCATION/openstack-ceph.yaml $LOCATION/openstack-lvm-ceph.yaml"
 
 # Change Ceph default features (if we want to attach using krbd)
 # echo -e "\nrbd default features = 3" | sudo tee -a /etc/ceph/ceph.conf
+if sudo podman container exists ceph; then
+  echo 'Ceph container exists reusing it'
+else
+  echo 'Starting ceph Pacific demo cluster'
+  sudo podman run -d --name ceph --net=host -e MON_IP=192.168.130.1 -e CEPH_PUBLIC_NETWORK=0.0.0.0/0 -e DEMO_DAEMONS='osd' quay.io/ceph/daemon:latest-pacific demo
+fi
 
-echo 'Running ceph Pacific demo cluster'
-sudo podman run -d --name ceph --net=host -v /etc/ceph:/etc/ceph:z -v /lib/modules:/lib/modules -e MON_IP=192.168.130.1 -e CEPH_PUBLIC_NETWORK=0.0.0.0/0 -e DEMO_DAEMONS='osd' quay.io/ceph/daemon:latest-pacific demo
+echo 'Waiting for Ceph config files to be created'
+until sudo podman exec -t ceph test -e /etc/ceph/I_AM_A_DEMO
+do
+  echo -n .
+  sleep 0.5
+done
+echo
 
-sleep 3
+TEMPDIR=`mktemp -d`
+trap 'sudo rm -rf -- "$TEMPDIR"' EXIT
+echo 'Copying Ceph config files from the container to $TEMPDIR'
+sudo podman cp ceph:/etc/ceph/ceph.conf $TEMPDIR
+sudo podman cp ceph:/etc/ceph/ceph.client.admin.keyring $TEMPDIR
+sudo chown `whoami` $TEMPDIR/*
+
+echo "Replacing openshift secret $SECRET_NAME"
+oc delete secret $SECRET_NAME 2>/dev/null || true
+oc create secret generic $SECRET_NAME --from-file=$TEMPDIR/ceph.conf --from-file=$TEMPDIR/ceph.client.admin.keyring
+
+FSID=`grep fsid $TEMPDIR/ceph.conf | cut -d' ' -f 3`
+for manifest in $OPENSTACK_YAMLS
+do
+  # echo "Replacing Ceph FSID in $OPENSTACK_YAML with $FSID"
+  echo "Replacing Ceph FSID in $manifest with $FSID"
+  sed -i "s/rbd_secret_uuid\\s*=.*/rbd_secret_uuid=$FSID/g" "$manifest"
+done
 
 sudo podman exec -it ceph bash -c 'ceph osd pool create volumes 4 && ceph osd pool application enable volumes rbd'
 sudo podman exec -it ceph bash -c 'ceph osd pool create backups 4 && ceph osd pool application enable backups rbd'

hack/dev/openstack-ceph.yaml

@@ -41,6 +41,13 @@ spec:
       customServiceConfig: |
         [DEFAULT]
         debug = true
+        enabled_backends=default_backend:rbd
+
+        [default_backend]
+        rbd_store_ceph_conf=/etc/ceph/ceph.conf
+        rbd_store_user=admin
+        rbd_store_pool=images
+        store_description=Ceph glance store backend.
       glanceAPIInternal:
         debug:
           service: false
@@ -51,14 +58,6 @@ spec:
           service: false
         preserveJobs: false
         replicas: 1
-      cephBackend:
-        cephFsid: 5fe62cc7-0392-4a32-8466-081ce0ea970f
-        cephMons: 192.168.130.1
-        cephClientKey: AQBCtBhj0gM6FRAACq4EGHK6qYqRBSbw4zFavg==
-        cephUser: admin
-        cephPools:
-          glance:
-            name: images
   cinder:
     enabled: true
     template:
@@ -99,16 +98,33 @@ spec:
           customServiceConfig: |
             [DEFAULT]
             enabled_backends=ceph
+            [ceph]
+            volume_backend_name=ceph
+            volume_driver=cinder.volume.drivers.rbd.RBDDriver
+            rbd_ceph_conf=/etc/ceph/ceph.conf
+            rbd_user=admin
+            rbd_pool=volumes
+            rbd_flatten_volume_from_snapshot=False
+            rbd_secret_uuid=5fe62cc7-0392-4a32-8466-081ce0ea970f
           debug:
             initContainer: false
             service: false
-      cephBackend:
-        cephFsid: 5fe62cc7-0392-4a32-8466-081ce0ea970f
-        cephMons: 192.168.130.1
-        cephClientKey: AQBCtBhj0gM6FRAACq4EGHK6qYqRBSbw4zFavg==
-        cephUser: admin
-        cephPools:
-          cinder:
-            name: volumes
-          cinder_backup:
-            name: backup
+  extraMounts:
+    - name: cephfiles
+      region: r1
+      extraVol:
+      - propagation:
+        - Glance
+        - CinderVolume
+        - CinderBackup
+        extraVolType: Ceph
+        volumes:
+        - name: ceph
+          projected:
+            sources:
+            - secret:
+                name: ceph-client-files
+        mounts:
+        - name: ceph
+          mountPath: "/etc/ceph"
+          readOnly: true

hack/dev/openstack-lvm-ceph.yaml

@@ -41,6 +41,13 @@ spec:
       customServiceConfig: |
         [DEFAULT]
         debug = true
+        enabled_backends=default_backend:rbd
+
+        [default_backend]
+        rbd_store_ceph_conf=/etc/ceph/ceph.conf
+        rbd_store_user=admin
+        rbd_store_pool=images
+        store_description=Ceph glance store backend.
       glanceAPIInternal:
         debug:
           service: false
@@ -51,14 +58,6 @@ spec:
           service: false
         preserveJobs: false
         replicas: 1
-      cephBackend:
-        cephFsid: 5fe62cc7-0392-4a32-8466-081ce0ea970f
-        cephMons: 192.168.130.1
-        cephClientKey: AQBCtBhj0gM6FRAACq4EGHK6qYqRBSbw4zFavg==
-        cephUser: admin
-        cephPools:
-          glance:
-            name: images
   cinder:
     enabled: true
     template:
@@ -116,16 +115,33 @@ spec:
           customServiceConfig: |
             [DEFAULT]
             enabled_backends=ceph
+            [ceph]
+            volume_backend_name=ceph
+            volume_driver=cinder.volume.drivers.rbd.RBDDriver
+            rbd_ceph_conf=/etc/ceph/ceph.conf
+            rbd_user=admin
+            rbd_pool=volumes
+            rbd_flatten_volume_from_snapshot=False
+            rbd_secret_uuid=5fe62cc7-0392-4a32-8466-081ce0ea970f
           debug:
             initContainer: false
             service: false
-      cephBackend:
-        cephFsid: 5fe62cc7-0392-4a32-8466-081ce0ea970f
-        cephMons: 192.168.130.1
-        cephClientKey: AQBCtBhj0gM6FRAACq4EGHK6qYqRBSbw4zFavg==
-        cephUser: admin
-        cephPools:
-          cinder:
-            name: volumes
-          cinder_backup:
-            name: backup
+  extraMounts:
+    - name: cephfiles
+      region: r1
+      extraVol:
+      - propagation:
+        - Glance
+        - CinderVolume
+        - CinderBackup
+        extraVolType: Ceph
+        volumes:
+        - name: ceph
+          projected:
+            sources:
+            - secret:
+                name: ceph-client-files
+        mounts:
+        - name: ceph
+          mountPath: "/etc/ceph"
+          readOnly: true