diff --git a/demo/csi/ceph-csi-plugin/README.md b/demo/csi/ceph-csi-plugin/README.md new file mode 100644 index 00000000000..267a4cd58b9 --- /dev/null +++ b/demo/csi/ceph-csi-plugin/README.md @@ -0,0 +1,66 @@ +# Openstack Ceph-CSI Plugin + +The configuration here is for the Ceph RBD driver, migrated from the k8s config [documentation](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-rbd.md). It can be easily modified for the CephFS Driver, as used [here](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-cephfs.md). + +## Requirements + +The example plugin job creates a file at `local/cloud.conf` using a [`template`](https://www.nomadproject.io/docs/job-specification/template) stanza which pulls the necessary credentials from a [Vault kv-v2](https://www.vaultproject.io/docs/secrets/kv/kv-v2) secrets store. + + +### Docker Privileged Mode + +The Ceph CSI Node task requires that [`privileged = true`](https://www.nomadproject.io/docs/drivers/docker#privileged) be set. This is not needed for the Controller task. + +## Container Arguments + +Refer to the official plugin [guide](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-rbd.md). + +- `--type=rbd` + + - Driver type `rbd` (or alternately `cephfs`) + +- `--endpoint=unix:///csi/csi.sock` + + - This option must match the `mount_dir` specified in the `csi_plugin` stanza for the task. + +- `--nodeid=${node.unique.name}` + + - A unique ID for the node the task is running on. Recommend using `${node.unique.name}` + +- `--cluster=${NOMAD_DC}` + + - The cluster the Controller/Node is a part of. Recommend using `${NOMAD_DC}` + +- `--instanceid=${attr.unique.platform.aws.instance-id}` + + - Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning. Used for topology-aware deployments. + +## Deployment + +### Plugin + +```bash +export NOMAD_ADDR=https://nomad.example.com:4646 +export NOMAD_TOKEN=34534-3sdf3-szfdsafsdf3423-zxdfsd3 +nomad job run ceph-csi-plugin.hcl +``` + +### Volume Registration + +The `external_id` value for the volume must be strictly formatted, see `ceph_csi.tf`. Based on [Ceph-CSI ID Format](https://github.com/ceph/ceph-csi/blob/71ddf51544be498eee03734573b765eb04480bb9/internal/util/volid.go#L27), see [examples](https://github.com/ceph/ceph-csi/blob/71ddf51544be498eee03734573b765eb04480bb9/internal/util/volid_test.go#L33). + +The `secrets` block will be populated with values pulled from `/etc/ceph/ceph.client..keyring`, e.g. +``` +userid = "" +userkey = "AWBg/BtfJInSFBATOrrnCh6UGE3QB3nYakdF+g==" +``` + +```bash +export NOMAD_ADDR=https://nomad.example.com:4646 +export NOMAD_TOKEN=34534-3sdf3-szfdsafsdf3423-zxdfsd3 +nomad volume register example_volume.hcl +``` + +## Ceph CSI Driver Source + +- https://github.com/ceph/ceph-csi diff --git a/demo/csi/ceph-csi-plugin/ceph-csi-id.tf b/demo/csi/ceph-csi-plugin/ceph-csi-id.tf new file mode 100644 index 00000000000..7ee7c13ebfc --- /dev/null +++ b/demo/csi/ceph-csi-plugin/ceph-csi-id.tf @@ -0,0 +1,33 @@ +locals { + # ClusterID: Is a unique ID per cluster that the CSI instance is serving and is restricted to + # lengths that can be accommodated in the encoding scheme. + # must be less than 128 chars. must match the cluster id in the csi plugin conf. + ClusterID = "" + + # EncodingVersion: Carries the version number of the encoding scheme used to encode the CSI ID, + # and is preserved for any future proofing w.r.t changes in the encoding scheme, and to retain + # ability to parse backward compatible encodings. + # https://github.com/ceph/ceph-csi/blob/ef1785ce4db0aa1f6878c770893bcabc71cff300/internal/cephfs/driver.go#L31 + EncodingVersion = 1 + + # LocationID: 64 bit integer identifier determining the location of the volume on the Ceph cluster. + # It is the ID of the poolname or fsname, for RBD or CephFS backed volumes respectively. + # see https://docs.ceph.com/docs/mimic/rbd/rados-rbd-cmds/ + LocationID = 7 + + # ObjectUUID: Is the on-disk uuid of the object (image/snapshot) name, for the CSI volume that + # corresponds to this CSI ID.. must be 36 chars long. + ObjectUUID = "abcd" +} + +data "template_file" "csi_id" { + template = "$${versionEncodedHex}-$${clusterIDLength}-$${ciClusterID}-$${poolIDEncodedHex}-$${ciObjectUUID}" + + vars = { + versionEncodedHex = "${format("%02X", local.EncodingVersion)}" + clusterIDLength = "${format("%02X", length(local.ClusterID))}" + ciClusterID = "${local.ClusterID}" + poolIDEncodedHex = "${format("%016X", local.LocationID)}" + ciObjectUUID = "${local.ObjectUUID}" + } +} diff --git a/demo/csi/ceph-csi-plugin/ceph-csi-plugin.hcl b/demo/csi/ceph-csi-plugin/ceph-csi-plugin.hcl new file mode 100644 index 00000000000..eb745460854 --- /dev/null +++ b/demo/csi/ceph-csi-plugin/ceph-csi-plugin.hcl @@ -0,0 +1,119 @@ +job "ceph-csi-plugin" { + datacenters = ["dc1"] + type = "system" + group "nodes" { + task "ceph-node" { + driver = "docker" + template { + data = <", + "monitors": [ + {{range $index, $service := service "mon.ceph"}}{{if gt $index 0}}, {{end}}"{{.Address}}"{{end}} + ] +}] +EOF + destination = "local/config.json" + change_mode = "restart" + } + config { + image = "quay.io/cephcsi/cephcsi:v2.1.2-amd64" + volumes = [ + "./local/config.json:/etc/ceph-csi-config/config.json" + ] + mounts = [ + { + type = "tmpfs" + target = "/tmp/csi/keys" + readonly = false + tmpfs_options { + size = 1000000 # size in bytes + } + } + ] + args = [ + "--type=rbd", + # Name of the driver + "--drivername=rbd.csi.ceph.com", + "--logtostderr", + "--nodeserver=true", + "--endpoint=unix://csi/csi.sock", + "--instanceid=${attr.unique.platform.aws.instance-id}", + "--nodeid=${attr.unique.consul.name}", + # TCP port for liveness metrics requests (/metrics) + "--metricsport=${NOMAD_PORT_prometheus}", + ] + privileged = true + resources { + cpu = 200 + memory = 500 + network { + mbits = 1 + // prometheus metrics port + port "prometheus" {} + } + } + } + service { + name = "prometheus" + port = "prometheus" + tags = ["ceph-csi"] + } + csi_plugin { + id = "ceph-csi" + type = "node" + mount_dir = "/csi" + } + } + task "ceph-controller" { + + template { + data = <", + "monitors": [ + {{range $index, $service := service "mon.ceph"}}{{if gt $index 0}}, {{end}}"{{.Address}}"{{end}} + ] +}] +EOF + destination = "local/config.json" + change_mode = "restart" + } + driver = "docker" + config { + image = "quay.io/cephcsi/cephcsi:v2.1.2-amd64" + volumes = [ + "./local/config.json:/etc/ceph-csi-config/config.json" + ] + resources { + cpu = 200 + memory = 500 + network { + mbits = 1 + // prometheus metrics port + port "prometheus" {} + } + } + args = [ + "--type=rbd", + "--controllerserver=true", + "--drivername=rbd.csi.ceph.com", + "--logtostderr", + "--endpoint=unix://csi/csi.sock", + "--metricsport=$${NOMAD_PORT_prometheus}", + "--nodeid=$${attr.unique.platform.aws.hostname}" + ] + } + service { + name = "prometheus" + port = "prometheus" + tags = ["ceph-csi"] + } + csi_plugin { + id = "ceph-csi" + type = "controller" + mount_dir = "/csi" + } + } + } +} \ No newline at end of file diff --git a/demo/csi/ceph-csi-plugin/example_volume.hcl b/demo/csi/ceph-csi-plugin/example_volume.hcl new file mode 100644 index 00000000000..636c31ed26d --- /dev/null +++ b/demo/csi/ceph-csi-plugin/example_volume.hcl @@ -0,0 +1,22 @@ +type = "csi" +id = "testvol" +name = "test_volume" +# this must be strictly formatted, see README +external_id = "ffff-0024-01616094-9d93-4178-bf45-c7eac19e8b15-000000000000ffff-00000000-1111-2222-bbbb-cacacacacaca" +access_mode = "single-node-writer" +attachment_mode = "block-device" +plugin_id = "ceph-csi" +mount_options { + fs_type = "ext4" +} +parameters {} +secrets { + userID = "" + userKey = "" +} +context { + # note: although these are 'parameters' in the ceph-csi spec + # they are passed through to the provider as 'context' + clusterID = "" + pool = "my_pool" +} \ No newline at end of file