From 1e65fcfbbe8c14dac208db8a966f845086f981ce Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 13 Aug 2024 14:49:00 -0400 Subject: [PATCH 1/7] incusd/instance/drivers/qmp: Export RunJSON Signed-off-by: Benjamin Somers --- .../server/instance/drivers/qmp/monitor.go | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/internal/server/instance/drivers/qmp/monitor.go b/internal/server/instance/drivers/qmp/monitor.go index 4182e940a56..8577a389c14 100644 --- a/internal/server/instance/drivers/qmp/monitor.go +++ b/internal/server/instance/drivers/qmp/monitor.go @@ -178,27 +178,13 @@ func (m *Monitor) ping() error { return nil } -// run executes a command. -func (m *Monitor) run(cmd string, args any, resp any) error { +// RunJSON executes a JSON-formatted command. +func (m *Monitor) RunJSON(request []byte, resp any) error { // Check if disconnected if m.disconnected { return ErrMonitorDisconnect } - // Run the command. - requestArgs := struct { - Execute string `json:"execute"` - Arguments any `json:"arguments,omitempty"` - }{ - Execute: cmd, - Arguments: args, - } - - request, err := json.Marshal(requestArgs) - if err != nil { - return err - } - out, err := m.qmp.Run(request) if err != nil { // Confirm the daemon didn't die. @@ -227,6 +213,25 @@ func (m *Monitor) run(cmd string, args any, resp any) error { return nil } +// run executes a command. +func (m *Monitor) run(cmd string, args any, resp any) error { + // Construct the command. + requestArgs := struct { + Execute string `json:"execute"` + Arguments any `json:"arguments,omitempty"` + }{ + Execute: cmd, + Arguments: args, + } + + request, err := json.Marshal(requestArgs) + if err != nil { + return err + } + + return m.RunJSON(request, resp) +} + // Connect creates or retrieves an existing QMP monitor for the path. func Connect(path string, serialCharDev string, eventHandler func(name string, data map[string]any)) (*Monitor, error) { monitorsLock.Lock() From 043359898b05d956531588277c90d5b97b801381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Tue, 13 Aug 2024 14:56:58 -0400 Subject: [PATCH 2/7] api: qemu_raw_qmp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- doc/api-extensions.md | 8 ++++++++ internal/version/api.go | 1 + 2 files changed, 9 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index f43f6b484f7..262cf668cd0 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -2554,3 +2554,11 @@ This introduces per-pool project disk limits, introducing a `limits.disk.pool.NA ## `network_ovn_isolated` This allows using `none` as the uplink network for an OVN network, making the network isolated. + +## `qemu_raw_qmp` + +This adds new configuration options to virtual machines to directly issue QMP commands at various stages of startup: + +* `raw.qemu.qmp.early` +* `raw.qemu.qmp.pre-start` +* `raw.qemu.qmp.post-start` diff --git a/internal/version/api.go b/internal/version/api.go index 4cc9d5c3f24..9c17bd28c67 100644 --- a/internal/version/api.go +++ b/internal/version/api.go @@ -433,6 +433,7 @@ var APIExtensions = []string{ "disk_volume_subpath", "projects_limits_disk_pool", "network_ovn_isolated", + "qemu_raw_qmp", } // APIExtensionsCount returns the number of available API extensions. From ed9284467644ea579b8b4741d6e5af486bf505fa Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 13 Aug 2024 14:49:32 -0400 Subject: [PATCH 3/7] incusd/instance: Add raw QMP config options Signed-off-by: Benjamin Somers --- internal/instance/config.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/internal/instance/config.go b/internal/instance/config.go index 541cc7db62e..88ed9f861b8 100644 --- a/internal/instance/config.go +++ b/internal/instance/config.go @@ -935,6 +935,33 @@ var InstanceConfigKeysVM = map[string]func(value string) error{ // shortdesc: Addition/override to the generated `qemu.conf` file "raw.qemu.conf": validate.IsAny, + // gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.early) + // + // --- + // type: blob + // liveupdate: no + // condition: virtual machine + // shortdesc: QMP commands to run before Incus QEMU initialization + "raw.qemu.qmp.early": validate.IsAny, + + // gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.post-start) + // + // --- + // type: blob + // liveupdate: no + // condition: virtual machine + // shortdesc: QMP commands to run after the VM has started + "raw.qemu.qmp.post-start": validate.IsAny, + + // gendoc:generate(entity=instance, group=raw, key=raw.qemu.qmp.pre-start) + // + // --- + // type: blob + // liveupdate: no + // condition: virtual machine + // shortdesc: QMP commands to run after Incus QEMU initialization and before the VM has started + "raw.qemu.qmp.pre-start": validate.IsAny, + // gendoc:generate(entity=instance, group=security, key=security.agent.metrics) // // --- From 6b697f743e29558274e5aa439e226946dac88f71 Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 13 Aug 2024 14:50:48 -0400 Subject: [PATCH 4/7] doc: Add QMP to wordlist Signed-off-by: Benjamin Somers --- doc/.wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/.wordlist.txt b/doc/.wordlist.txt index 06755106f84..a72e2bf83e8 100644 --- a/doc/.wordlist.txt +++ b/doc/.wordlist.txt @@ -211,6 +211,7 @@ Podman PTS qdisc QEMU +QMP qgroup qgroups RADOS From c83dbb2911042f15ec60cf0c5963e0a6d832e87d Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 13 Aug 2024 14:51:03 -0400 Subject: [PATCH 5/7] doc: Update configs Signed-off-by: Benjamin Somers --- doc/config_options.txt | 24 ++++++++++++++++++ internal/server/metadata/configuration.json | 27 +++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/doc/config_options.txt b/doc/config_options.txt index 870bfa95b1e..47d8f6e1d8f 100644 --- a/doc/config_options.txt +++ b/doc/config_options.txt @@ -681,6 +681,30 @@ For example: `both 1000 1000` See {ref}`instance-options-qemu` for more information. ``` +```{config:option} raw.qemu.qmp.early instance-raw +:condition: "virtual machine" +:liveupdate: "no" +:shortdesc: "QMP commands to run before Incus QEMU initialization" +:type: "blob" + +``` + +```{config:option} raw.qemu.qmp.post-start instance-raw +:condition: "virtual machine" +:liveupdate: "no" +:shortdesc: "QMP commands to run after the VM has started" +:type: "blob" + +``` + +```{config:option} raw.qemu.qmp.pre-start instance-raw +:condition: "virtual machine" +:liveupdate: "no" +:shortdesc: "QMP commands to run after Incus QEMU initialization and before the VM has started" +:type: "blob" + +``` + ```{config:option} raw.seccomp instance-raw :condition: "container" :liveupdate: "no" diff --git a/internal/server/metadata/configuration.json b/internal/server/metadata/configuration.json index 7f9dd8da5d6..362b6270ad0 100644 --- a/internal/server/metadata/configuration.json +++ b/internal/server/metadata/configuration.json @@ -752,6 +752,33 @@ "type": "blob" } }, + { + "raw.qemu.qmp.early": { + "condition": "virtual machine", + "liveupdate": "no", + "longdesc": "", + "shortdesc": "QMP commands to run before Incus QEMU initialization", + "type": "blob" + } + }, + { + "raw.qemu.qmp.post-start": { + "condition": "virtual machine", + "liveupdate": "no", + "longdesc": "", + "shortdesc": "QMP commands to run after the VM has started", + "type": "blob" + } + }, + { + "raw.qemu.qmp.pre-start": { + "condition": "virtual machine", + "liveupdate": "no", + "longdesc": "", + "shortdesc": "QMP commands to run after Incus QEMU initialization and before the VM has started", + "type": "blob" + } + }, { "raw.seccomp": { "condition": "container", From eba841813ece9517331454fce2ca625aedd6dbbd Mon Sep 17 00:00:00 2001 From: Benjamin Somers Date: Tue, 13 Aug 2024 14:50:26 -0400 Subject: [PATCH 6/7] incusd/instance/qemu: Add QMP hooks Signed-off-by: Benjamin Somers --- .../server/instance/drivers/driver_qemu.go | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index ff265aec67b..d96955729f3 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -1110,6 +1110,29 @@ func (d *qemu) Start(stateful bool) error { return d.start(stateful, nil) } +func (d *qemu) runQMP(monitor *qmp.Monitor, stage string) error { + commands, ok := d.expandedConfig["raw.qemu.qmp."+stage] + if ok { + var commandList []map[string]any + err := json.Unmarshal([]byte(commands), &commandList) + if err != nil { + err = fmt.Errorf("Failed to parse QMP commands at %s stage (expected JSON list of objects): %w", stage, err) + return err + } + + for _, command := range commandList { + jsonCommand, _ := json.Marshal(command) + err = monitor.RunJSON(jsonCommand, nil) + if err != nil { + err = fmt.Errorf("Failed to run QMP command %s at %s stage: %w", jsonCommand, stage, err) + return err + } + } + } + + return nil +} + // start starts the instance and can use an existing InstanceOperation lock. func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { d.logger.Debug("Start started", logger.Ctx{"stateful": stateful}) @@ -1700,6 +1723,13 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { // onStop hook isn't triggered prematurely (as this function's reverter will clean up on failure to start). monitor.SetOnDisconnectEvent(false) + // Early QMP hook + err = d.runQMP(monitor, "early") + if err != nil { + op.Done(err) + return err + } + // Apply CPU pinning. if cpuInfo.vcpus == nil { if d.architectureSupportsCPUHotplug() && cpuInfo.cores > 1 { @@ -1757,6 +1787,13 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { } } + // Pre-start QMP hook + err = d.runQMP(monitor, "pre-start") + if err != nil { + op.Done(err) + return err + } + // Due to a bug in QEMU, devices added using QMP's device_add command do not have their bootindex option // respected (even if added before emuation is started). To workaround this we must reset the VM in order // for it to rebuild its boot config and to take into account the devices bootindex settings. @@ -1825,6 +1862,16 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error { revert.Success() + // Post-start QMP hook + err = d.runQMP(monitor, "post-start") + if err != nil { + op.Done(err) + + // Shut down the VM if the post-start commands fail. + _ = d.Stop(false) + return err + } + // Run any post-start hooks. err = d.runHooks(postStartHooks) if err != nil { From e731940a565bd31ecad8883ea2429dcf269783f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= Date: Tue, 13 Aug 2024 14:54:12 -0400 Subject: [PATCH 7/7] incusd/project: Update low-level properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber --- internal/server/project/permissions.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/server/project/permissions.go b/internal/server/project/permissions.go index 37ba737ea07..1d36fc28fce 100644 --- a/internal/server/project/permissions.go +++ b/internal/server/project/permissions.go @@ -902,8 +902,13 @@ func isVMLowLevelOptionForbidden(key string) bool { "boot.host_shutdown_action", "boot.host_shutdown_timeout", "limits.memory.hugepages", + "raw.apparmor", "raw.idmap", "raw.qemu", + "raw.qemu.conf", + "raw.qemu.qmp.early", + "raw.qemu.qmp.post-start", + "raw.qemu.qmp.pre-start", }, key) }