Add setup to emulate NVMe device

The setup allows to create and emulate in the provisioned nodes an NVMe device. This can be particularly useful for testing scenarios with NVMe like NVMe passthrough. Added in the gocli provision the new flag nvme in order to create the device. Signed-off-by: Alice Frosi <[email protected]>
kubevirt · Sep 6, 2021 · adb93c6 · adb93c6
1 parent 576bc8c
commit adb93c6
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 1 deletion.
diff --git a/cluster-provision/centos8/scripts/vm.sh b/cluster-provision/centos8/scripts/vm.sh
@@ -20,6 +20,7 @@ while true; do
     -n | --next-disk ) NEXT_DISK="$2"; shift 2 ;;
     -b | --block-device ) BLOCK_DEV="$2"; shift 2 ;;
     -s | --block-device-size ) BLOCK_DEV_SIZE="$2"; shift 2 ;;
+    -n | --nvme-device-size ) NVME_DISK_SIZES+="$2 "; shift 2 ;;
     -- ) shift; break ;;
     * ) break ;;
   esac
@@ -108,6 +109,14 @@ if [ -n "${BLOCK_DEV}" ]; then
   block_dev_arg="-drive format=qcow2,file=${BLOCK_DEV},if=virtio,cache=unsafe"
 fi
 
+disk_num=0
+for size in ${NVME_DISK_SIZES[@]}; do
+  echo "Creating disk "$size" for NVMe disk emulation"
+  disk="/nvme-"${disk_num}".img"
+  qemu-img create -f raw $disk $size
+  let "disk_num+=1"
+done
+
 exec qemu-system-x86_64 -enable-kvm -drive format=qcow2,file=${next},if=virtio,cache=unsafe ${block_dev_arg} \
   -device virtio-net-pci,netdev=network0,mac=52:55:00:d1:55:${n} \
   -netdev tap,id=network0,ifname=tap${n},script=no,downscript=no \

diff --git a/cluster-provision/gocli/cmd/run.go b/cluster-provision/gocli/cmd/run.go
@@ -49,8 +49,10 @@ systemctl restart docker
 EOF
 `
 const etcdDataDir = "/var/lib/etcd"
+const nvmeDiskImagePrefix = "/nvme"
 
 var cli *client.Client
+var nvmeDisks []string
 
 type dockerSetting struct {
 	Proxy string
@@ -96,6 +98,7 @@ func NewRunCommand() *cobra.Command {
 	run.Flags().String("container-org", "kubevirtci", "the organization at the registry to pull the container from")
 	run.Flags().String("container-suffix", "", "Override container suffix stored at the cli binary")
 	run.Flags().String("gpu", "", "pci address of a GPU to assign to a node")
+	run.Flags().StringArrayVar(&nvmeDisks, "nvme", []string{}, "size of the emulate NVMe disk to pass to the node")
 	run.Flags().Bool("run-etcd-on-memory", false, "configure etcd to run on RAM memory, etcd data will not be persistent")
 	run.Flags().String("etcd-capacity", "512M", "set etcd data mount size.\nthis flag takes affect only when 'run-etcd-on-memory' is specified")
 
@@ -482,6 +485,16 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
 			nodeQemuArgs = fmt.Sprintf("%s -device vfio-pci,host=%s", nodeQemuArgs, gpuAddress)
 		}
 
+		var vmArgsNvmeDisks []string
+		if len(nvmeDisks) > 0 {
+			for i, size := range nvmeDisks {
+				resource.MustParse(size)
+				disk := fmt.Sprintf("%s-%d.img", nvmeDiskImagePrefix, i)
+				nodeQemuArgs = fmt.Sprintf("%s -drive file=%s,format=raw,id=NVME%d,if=none -device nvme,drive=NVME%d,serial=nvme-%d", nodeQemuArgs, disk, i, i, i)
+				vmArgsNvmeDisks = append(vmArgsNvmeDisks, fmt.Sprintf("--nvme-device-size %s", size))
+			}
+		}
+
 		additionalArgs := []string{}
 		if len(nodeQemuArgs) > 0 {
 			additionalArgs = append(additionalArgs, "--qemu-args", shellescape.Quote(nodeQemuArgs))
@@ -500,7 +513,7 @@ func run(cmd *cobra.Command, args []string) (retErr error) {
 			Env: []string{
 				fmt.Sprintf("NODE_NUM=%s", nodeNum),
 			},
-			Cmd: []string{"/bin/bash", "-c", fmt.Sprintf("/vm.sh -n /var/run/disk/disk.qcow2 --memory %s --cpu %s %s %s", memory, strconv.Itoa(int(cpu)), blockDev, strings.Join(additionalArgs, " "))},
+			Cmd: []string{"/bin/bash", "-c", fmt.Sprintf("/vm.sh -n /var/run/disk/disk.qcow2 --memory %s --cpu %s %s %s %s", memory, strconv.Itoa(int(cpu)), blockDev, strings.Join(vmArgsNvmeDisks, " "), strings.Join(additionalArgs, " "))},
 		}
 
 		if cephEnabled {