Merge pull request #1115 from bensmrs/main

Add support for manual QMP configuration

doc/.wordlist.txt

@@ -211,6 +211,7 @@ Podman
 PTS
 qdisc
 QEMU
+QMP
 qgroup
 qgroups
 RADOS

doc/api-extensions.md

@@ -2554,3 +2554,11 @@ This introduces per-pool project disk limits, introducing a `limits.disk.pool.NA
 ## `network_ovn_isolated`
 
 This allows using `none` as the uplink network for an OVN network, making the network isolated.
+
+## `qemu_raw_qmp`
+
+This adds new configuration options to virtual machines to directly issue QMP commands at various stages of startup:
+
+* `raw.qemu.qmp.early`
+* `raw.qemu.qmp.pre-start`
+* `raw.qemu.qmp.post-start`

doc/config_options.txt

@@ -681,6 +681,30 @@ For example: `both 1000 1000`
 See {ref}`instance-options-qemu` for more information.
 ```
 
+```{config:option} raw.qemu.qmp.early instance-raw
+:condition: "virtual machine"
+:liveupdate: "no"
+:shortdesc: "QMP commands to run before Incus QEMU initialization"
+:type: "blob"
+
+```
+
+```{config:option} raw.qemu.qmp.post-start instance-raw
+:condition: "virtual machine"
+:liveupdate: "no"
+:shortdesc: "QMP commands to run after the VM has started"
+:type: "blob"
+
+```
+
+```{config:option} raw.qemu.qmp.pre-start instance-raw
+:condition: "virtual machine"
+:liveupdate: "no"
+:shortdesc: "QMP commands to run after Incus QEMU initialization and before the VM has started"
+:type: "blob"
+
+```
+
 ```{config:option} raw.seccomp instance-raw
 :condition: "container"
 :liveupdate: "no"

internal/instance/config.go

@@ -935,6 +935,33 @@ var InstanceConfigKeysVM = map[string]func(value string) error{
 	//  shortdesc: Addition/override to the generated `qemu.conf` file
 	"raw.qemu.conf": validate.IsAny,
 
+	// gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.early)
+	//
+	// ---
+	//  type: blob
+	//  liveupdate: no
+	//  condition: virtual machine
+	//  shortdesc: QMP commands to run before Incus QEMU initialization
+	"raw.qemu.qmp.early": validate.IsAny,
+
+	// gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.post-start)
+	//
+	// ---
+	//  type: blob
+	//  liveupdate: no
+	//  condition: virtual machine
+	//  shortdesc: QMP commands to run after the VM has started
+	"raw.qemu.qmp.post-start": validate.IsAny,
+
+	// gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.pre-start)
+	//
+	// ---
+	//  type: blob
+	//  liveupdate: no
+	//  condition: virtual machine
+	//  shortdesc: QMP commands to run after Incus QEMU initialization and before the VM has started
+	"raw.qemu.qmp.pre-start": validate.IsAny,
+
 	// gendoc:generate(entity=instance, group=security, key=security.agent.metrics)
 	//
 	// ---

internal/server/instance/drivers/driver_qemu.go

@@ -1110,6 +1110,29 @@ func (d *qemu) Start(stateful bool) error {
 	return d.start(stateful, nil)
 }
 
+func (d *qemu) runQMP(monitor *qmp.Monitor, stage string) error {
+	commands, ok := d.expandedConfig["raw.qemu.qmp."+stage]
+	if ok {
+		var commandList []map[string]any
+		err := json.Unmarshal([]byte(commands), &commandList)
+		if err != nil {
+			err = fmt.Errorf("Failed to parse QMP commands at %s stage (expected JSON list of objects): %w", stage, err)
+			return err
+		}
+
+		for _, command := range commandList {
+			jsonCommand, _ := json.Marshal(command)
+			err = monitor.RunJSON(jsonCommand, nil)
+			if err != nil {
+				err = fmt.Errorf("Failed to run QMP command %s at %s stage: %w", jsonCommand, stage, err)
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // start starts the instance and can use an existing InstanceOperation lock.
 func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error {
 	d.logger.Debug("Start started", logger.Ctx{"stateful": stateful})
@@ -1700,6 +1723,13 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error {
 	// onStop hook isn't triggered prematurely (as this function's reverter will clean up on failure to start).
 	monitor.SetOnDisconnectEvent(false)
 
+	// Early QMP hook
+	err = d.runQMP(monitor, "early")
+	if err != nil {
+		op.Done(err)
+		return err
+	}
+
 	// Apply CPU pinning.
 	if cpuInfo.vcpus == nil {
 		if d.architectureSupportsCPUHotplug() && cpuInfo.cores > 1 {
@@ -1757,6 +1787,13 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error {
 		}
 	}
 
+	// Pre-start QMP hook
+	err = d.runQMP(monitor, "pre-start")
+	if err != nil {
+		op.Done(err)
+		return err
+	}
+
 	// Due to a bug in QEMU, devices added using QMP's device_add command do not have their bootindex option
 	// respected (even if added before emuation is started). To workaround this we must reset the VM in order
 	// for it to rebuild its boot config and to take into account the devices bootindex settings.
@@ -1825,6 +1862,16 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error {
 
 	revert.Success()
 
+	// Post-start QMP hook
+	err = d.runQMP(monitor, "post-start")
+	if err != nil {
+		op.Done(err)
+
+		// Shut down the VM if the post-start commands fail.
+		_ = d.Stop(false)
+		return err
+	}
+
 	// Run any post-start hooks.
 	err = d.runHooks(postStartHooks)
 	if err != nil {

internal/server/instance/drivers/qmp/monitor.go

@@ -178,27 +178,13 @@ func (m *Monitor) ping() error {
 	return nil
 }
 
-// run executes a command.
-func (m *Monitor) run(cmd string, args any, resp any) error {
+// RunJSON executes a JSON-formatted command.
+func (m *Monitor) RunJSON(request []byte, resp any) error {
 	// Check if disconnected
 	if m.disconnected {
 		return ErrMonitorDisconnect
 	}
 
-	// Run the command.
-	requestArgs := struct {
-		Execute   string `json:"execute"`
-		Arguments any    `json:"arguments,omitempty"`
-	}{
-		Execute:   cmd,
-		Arguments: args,
-	}
-
-	request, err := json.Marshal(requestArgs)
-	if err != nil {
-		return err
-	}
-
 	out, err := m.qmp.Run(request)
 	if err != nil {
 		// Confirm the daemon didn't die.
@@ -227,6 +213,25 @@ func (m *Monitor) run(cmd string, args any, resp any) error {
 	return nil
 }
 
+// run executes a command.
+func (m *Monitor) run(cmd string, args any, resp any) error {
+	// Construct the command.
+	requestArgs := struct {
+		Execute   string `json:"execute"`
+		Arguments any    `json:"arguments,omitempty"`
+	}{
+		Execute:   cmd,
+		Arguments: args,
+	}
+
+	request, err := json.Marshal(requestArgs)
+	if err != nil {
+		return err
+	}
+
+	return m.RunJSON(request, resp)
+}
+
 // Connect creates or retrieves an existing QMP monitor for the path.
 func Connect(path string, serialCharDev string, eventHandler func(name string, data map[string]any)) (*Monitor, error) {
 	monitorsLock.Lock()

internal/server/metadata/configuration.json

@@ -752,6 +752,33 @@
 							"type": "blob"
 						}
 					},
+					{
+						"raw.qemu.qmp.early": {
+							"condition": "virtual machine",
+							"liveupdate": "no",
+							"longdesc": "",
+							"shortdesc": "QMP commands to run before Incus QEMU initialization",
+							"type": "blob"
+						}
+					},
+					{
+						"raw.qemu.qmp.post-start": {
+							"condition": "virtual machine",
+							"liveupdate": "no",
+							"longdesc": "",
+							"shortdesc": "QMP commands to run after the VM has started",
+							"type": "blob"
+						}
+					},
+					{
+						"raw.qemu.qmp.pre-start": {
+							"condition": "virtual machine",
+							"liveupdate": "no",
+							"longdesc": "",
+							"shortdesc": "QMP commands to run after Incus QEMU initialization and before the VM has started",
+							"type": "blob"
+						}
+					},
 					{
 						"raw.seccomp": {
 							"condition": "container",

internal/server/project/permissions.go

@@ -902,8 +902,13 @@ func isVMLowLevelOptionForbidden(key string) bool {
 		"boot.host_shutdown_action",
 		"boot.host_shutdown_timeout",
 		"limits.memory.hugepages",
+		"raw.apparmor",
 		"raw.idmap",
 		"raw.qemu",
+		"raw.qemu.conf",
+		"raw.qemu.qmp.early",
+		"raw.qemu.qmp.post-start",
+		"raw.qemu.qmp.pre-start",
 	},
 		key)
 }

internal/version/api.go

@@ -433,6 +433,7 @@ var APIExtensions = []string{
 	"disk_volume_subpath",
 	"projects_limits_disk_pool",
 	"network_ovn_isolated",
+	"qemu_raw_qmp",
 }
 
 // APIExtensionsCount returns the number of available API extensions.