From f488d9890cb9d3a7c3ba1be37b086e3d9af7d1a1 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Tue, 4 Apr 2023 15:00:53 -0500 Subject: [PATCH] Add support for HVSOCK on hyperv Windows HyperV uses HVSocks (Windows adaptation of vsock) for communicating between vms and the host. Podman machine in Qemu uses a virtual UDS to signal the host that the machine is booted. In HyperV, we can use a HVSOCK for the same purpose. One of the big aspects of using HVSOCK on Windows is that the HVSOCK must be entered into the Windows registry. So now part of init and rm of a podman machine, entries must be added and removed respectively. Also duplicates are a no-no. Signed-off-by: Brent Baude --- go.mod | 2 +- pkg/machine/hyperv/config.go | 9 +- pkg/machine/hyperv/machine.go | 83 ++++++++--- pkg/machine/hyperv/vsock.go | 270 ++++++++++++++++++++++++++++++++++ pkg/machine/ignition.go | 77 ++++------ pkg/machine/qemu/machine.go | 31 +++- 6 files changed, 403 insertions(+), 69 deletions(-) create mode 100644 pkg/machine/hyperv/vsock.go diff --git a/go.mod b/go.mod index 7e256ac256..64c782b74d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( github.com/BurntSushi/toml v1.2.1 + github.com/Microsoft/go-winio v0.6.0 github.com/blang/semver/v4 v4.0.0 github.com/buger/goterm v1.0.4 github.com/checkpoint-restore/checkpointctl v0.1.0 @@ -74,7 +75,6 @@ require ( require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect diff --git a/pkg/machine/hyperv/config.go b/pkg/machine/hyperv/config.go index 04430e545c..8d334a4deb 100644 --- a/pkg/machine/hyperv/config.go +++ b/pkg/machine/hyperv/config.go @@ -6,7 +6,6 @@ package hyperv import ( "encoding/json" "errors" - "github.com/sirupsen/logrus" "io/fs" "os" "path/filepath" @@ -15,6 +14,7 @@ import ( "github.com/containers/libhvee/pkg/hypervctl" "github.com/containers/podman/v4/pkg/machine" "github.com/docker/go-units" + "github.com/sirupsen/logrus" ) type Virtualization struct { @@ -178,7 +178,6 @@ func (v Virtualization) NewMachine(opts machine.InitOptions) (machine.VM, error) return nil, err } return v.LoadVMByName(opts.Name) - } func (v Virtualization) RemoveAndCleanMachines() error { @@ -221,6 +220,12 @@ func (v Virtualization) RemoveAndCleanMachines() error { if err := vm.Remove(mm.ImagePath.GetPath()); err != nil { prevErr = handlePrevError(err, prevErr) } + if err := mm.ReadyHVSock.Remove(); err != nil { + prevErr = handlePrevError(err, prevErr) + } + if err := mm.NetworkHVSock.Remove(); err != nil { + prevErr = handlePrevError(err, prevErr) + } } // Nuke the config and dataDirs diff --git a/pkg/machine/hyperv/machine.go b/pkg/machine/hyperv/machine.go index 177dc8c8a1..06e4388137 100644 --- a/pkg/machine/hyperv/machine.go +++ b/pkg/machine/hyperv/machine.go @@ -56,12 +56,8 @@ const ( ) type HyperVMachine struct { - // copied from qemu, cull and add as needed - // ConfigPath is the fully qualified path to the configuration file ConfigPath machine.VMFile - // The command line representation of the qemu command - //CmdLine []string // HostUser contains info about host user machine.HostUser // ImageConfig describes the bootable image @@ -70,14 +66,10 @@ type HyperVMachine struct { Mounts []machine.Mount // Name of VM Name string - // PidFilePath is the where the Proxy PID file lives - //PidFilePath machine.VMFile - // VMPidFilePath is the where the VM PID file lives - //VMPidFilePath machine.VMFile - // QMPMonitor is the qemu monitor object for sending commands - //QMPMonitor Monitor + // NetworkVSock is for the user networking + NetworkHVSock HVSockRegistryEntry // ReadySocket tells host when vm is booted - ReadySocket machine.VMFile + ReadyHVSock HVSockRegistryEntry // ResourceConfig is physical attrs of the VM machine.ResourceConfig // SSHConfig for accessing the remote vm @@ -95,6 +87,18 @@ func (m *HyperVMachine) Init(opts machine.InitOptions) (bool, error) { key string ) + // Add the network and ready sockets to the Windows registry + networkHVSock, err := NewHVSockRegistryEntry(m.Name, Network) + if err != nil { + return false, err + } + eventHVSocket, err := NewHVSockRegistryEntry(m.Name, Events) + if err != nil { + return false, err + } + m.NetworkHVSock = *networkHVSock + m.ReadyHVSock = *eventHVSocket + sshDir := filepath.Join(homedir.Get(), ".ssh") m.IdentityPath = filepath.Join(sshDir, m.Name) @@ -170,12 +174,39 @@ func (m *HyperVMachine) Init(opts machine.InitOptions) (bool, error) { Name: user, Key: key, VMName: m.Name, + VMType: machine.HyperVVirt, TimeZone: opts.TimeZone, WritePath: m.IgnitionFile.GetPath(), UID: m.UID, } - if err := machine.NewIgnitionFile(ign, machine.HyperVVirt); err != nil { + if err := ign.GenerateIgnitionConfig(); err != nil { + return false, err + } + + // ready is a unit file that sets up the virtual serial device + // where when the VM is done configuring, it will send an ack + // so a listening host knows it can being interacting with it + // + // VSOCK-CONNECT:2 <- shortcut to connect to the hostvm + ready := `[Unit] +After=remove-moby.service sshd.socket sshd.service +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo Ready | socat - VSOCK-CONNECT:2:%d' +[Install] +RequiredBy=default.target +` + readyUnit := machine.Unit{ + Enabled: machine.BoolToPtr(true), + Name: "ready.service", + Contents: machine.StrToPtr(fmt.Sprintf(ready, m.ReadyHVSock.Port)), + } + ign.Cfg.Systemd.Units = append(ign.Cfg.Systemd.Units, readyUnit) + if err := ign.Write(); err != nil { return false, err } // The ignition file has been written. We now need to @@ -259,12 +290,6 @@ func (m *HyperVMachine) Remove(_ string, opts machine.RemoveOptions) (string, fu files = append(files, diskPath) } - if err := machine.RemoveConnection(m.Name); err != nil { - logrus.Error(err) - } - if err := machine.RemoveConnection(m.Name + "-root"); err != nil { - logrus.Error(err) - } files = append(files, getVMConfigPath(m.ConfigPath.GetPath(), m.Name)) confirmationMessage := "\nThe following files will be deleted:\n\n" for _, msg := range files { @@ -278,6 +303,22 @@ func (m *HyperVMachine) Remove(_ string, opts machine.RemoveOptions) (string, fu logrus.Error(err) } } + if err := machine.RemoveConnection(m.Name); err != nil { + logrus.Error(err) + } + if err := machine.RemoveConnection(m.Name + "-root"); err != nil { + logrus.Error(err) + } + + // Remove the HVSOCK for networking + if err := m.NetworkHVSock.Remove(); err != nil { + logrus.Errorf("unable to remove registry entry for %s: %q", m.NetworkHVSock.KeyName, err) + } + + // Remove the HVSOCK for events + if err := m.ReadyHVSock.Remove(); err != nil { + logrus.Errorf("unable to remove registry entry for %s: %q", m.NetworkHVSock.KeyName, err) + } return vm.Remove(diskPath) }, nil } @@ -360,7 +401,11 @@ func (m *HyperVMachine) Start(name string, opts machine.StartOptions) error { if vm.State() != hypervctl.Disabled { return hypervctl.ErrMachineStateInvalid } - return vm.Start() + if err := vm.Start(); err != nil { + return err + } + // Wait on notification from the guest + return m.ReadyHVSock.Listen() } func (m *HyperVMachine) State(_ bool) (machine.Status, error) { diff --git a/pkg/machine/hyperv/vsock.go b/pkg/machine/hyperv/vsock.go new file mode 100644 index 0000000000..f4bcbc9358 --- /dev/null +++ b/pkg/machine/hyperv/vsock.go @@ -0,0 +1,270 @@ +//go:build windows +// +build windows + +package hyperv + +import ( + "bufio" + "errors" + "fmt" + "strings" + + "github.com/Microsoft/go-winio" + "github.com/containers/podman/v4/utils" + "github.com/sirupsen/logrus" + "golang.org/x/sys/windows/registry" +) + +var ErrVSockRegistryEntryExists = errors.New("registry entry already exists") + +const ( + // HvsockMachineName is the string identifier for the machine name in a registry entry + HvsockMachineName = "MachineName" + // HvsockPurpose is the string identifier for the sock purpose in a registry entry + HvsockPurpose = "Purpose" + // VsockRegistryPath describes the registry path to where the hvsock registry entries live + VsockRegistryPath = `SOFTWARE\Microsoft\Windows NT\CurrentVersion\Virtualization\GuestCommunicationServices` + // LinuxVm is the default guid for a Linux VM on Windows + LinuxVm = "FACB-11E6-BD58-64006A7986D3" +) + +// HVSockPurpose describes what the hvsock is needed for +type HVSockPurpose int + +const ( + // Network implies the sock is used for user-mode networking + Network HVSockPurpose = iota + // Events implies the sock is used for notification (like "Ready") + Events +) + +func (hv HVSockPurpose) string() string { + switch hv { + case Network: + return "Network" + case Events: + return "Events" + } + return "" +} + +func (hv HVSockPurpose) Equal(purpose string) bool { + return hv.string() == purpose +} + +func toHVSockPurpose(p string) (HVSockPurpose, error) { + switch p { + case "Network": + return Network, nil + case "Events": + return Events, nil + } + return 0, fmt.Errorf("unknown hvsockpurpose: %s", p) +} + +func openVSockRegistryEntry(entry string) (registry.Key, error) { + return registry.OpenKey(registry.LOCAL_MACHINE, entry, registry.QUERY_VALUE) +} + +// HVSockRegistryEntry describes a registry entry used in Windows for HVSOCK implementations +type HVSockRegistryEntry struct { + KeyName string `json:"key_name"` + Purpose HVSockPurpose `json:"purpose"` + Port uint64 `json:"port"` + MachineName string `json:"machineName"` + Key registry.Key `json:"key,omitempty"` +} + +// Add creates a new Windows registry entry with string values from the +// HVSockRegistryEntry. +func (hv *HVSockRegistryEntry) Add() error { + if err := hv.validate(); err != nil { + return err + } + exists, err := hv.exists() + if err != nil { + return err + } + if exists { + return fmt.Errorf("%q: %s", ErrVSockRegistryEntryExists, hv.KeyName) + } + parentKey, err := registry.OpenKey(registry.LOCAL_MACHINE, VsockRegistryPath, registry.QUERY_VALUE) + defer func() { + if err := parentKey.Close(); err != nil { + logrus.Error(err) + } + }() + if err != nil { + return err + } + newKey, _, err := registry.CreateKey(parentKey, hv.KeyName, registry.WRITE) + defer func() { + if err := newKey.Close(); err != nil { + logrus.Error(err) + } + }() + if err != nil { + return err + } + + if err := newKey.SetStringValue(HvsockPurpose, hv.Purpose.string()); err != nil { + return err + } + return newKey.SetStringValue(HvsockMachineName, hv.MachineName) +} + +// Remove deletes the registry key and its string values +func (hv *HVSockRegistryEntry) Remove() error { + return registry.DeleteKey(registry.LOCAL_MACHINE, hv.fqPath()) +} + +func (hv *HVSockRegistryEntry) fqPath() string { + return fmt.Sprintf("%s\\%s", VsockRegistryPath, hv.KeyName) +} + +func (hv *HVSockRegistryEntry) validate() error { + if hv.Port < 1 { + return errors.New("port must be larger than 1") + } + if len(hv.Purpose.string()) < 1 { + return errors.New("required field purpose is empty") + } + if len(hv.MachineName) < 1 { + return errors.New("required field machinename is empty") + } + if len(hv.KeyName) < 1 { + return errors.New("required field keypath is empty") + } + //decimal_num, err = strconv.ParseInt(hexadecimal_num, 16, 64) + return nil +} + +func (hv *HVSockRegistryEntry) exists() (bool, error) { + foo := hv.fqPath() + _ = foo + _, err := openVSockRegistryEntry(hv.fqPath()) + if err == nil { + return true, err + } + if errors.Is(err, registry.ErrNotExist) { + return false, nil + } + return false, err +} + +// findOpenHVSockPort looks for an open random port. it verifies the port is not +// already being used by another hvsock in the Windows registry. +func findOpenHVSockPort() (uint64, error) { + // If we cannot find a free port in 10 attempts, something is wrong + for i := 0; i < 10; i++ { + port, err := utils.GetRandomPort() + if err != nil { + return 0, err + } + // Try and load registry entries by port to see if they exist + _, err = LoadHVSockRegistryEntry(uint64(port)) + if err == nil { + // the port is no good, it is being used; try again + logrus.Errorf("port %d is already used for hvsock", port) + continue + } + if errors.Is(err, registry.ErrNotExist) { + // the port is good to go + return uint64(port), nil + } + if err != nil { + // something went wrong + return 0, err + } + } + return 0, errors.New("unable to find a free port for hvsock use") +} + +// NewHVSockRegistryEntry is a constructor to make a new registry entry in Windows. After making the new +// object, you must call the add() method to *actually* add it to the Windows registry. +func NewHVSockRegistryEntry(machineName string, purpose HVSockPurpose) (*HVSockRegistryEntry, error) { + // a so-called wildcard entry ... everything from FACB -> 6D3 is MS special sauce + // for a " linux vm". this first segment is hexi for the hvsock port number + //00000400-FACB-11E6-BD58-64006A7986D3 + port, err := findOpenHVSockPort() + if err != nil { + return nil, err + } + r := HVSockRegistryEntry{ + KeyName: portToKeyName(port), + Purpose: purpose, + Port: port, + MachineName: machineName, + } + if err := r.Add(); err != nil { + return nil, err + } + return &r, nil +} + +func portToKeyName(port uint64) string { + // this could be flattened but given the complexity, I thought it might + // be more difficult to read + hexi := strings.ToUpper(fmt.Sprintf("%08x", port)) + return fmt.Sprintf("%s-%s", hexi, LinuxVm) +} + +func LoadHVSockRegistryEntry(port uint64) (*HVSockRegistryEntry, error) { + keyName := portToKeyName(port) + fqPath := fmt.Sprintf("%s\\%s", VsockRegistryPath, keyName) + k, err := openVSockRegistryEntry(fqPath) + if err != nil { + return nil, err + } + p, _, err := k.GetStringValue(HvsockPurpose) + if err != nil { + return nil, err + } + + purpose, err := toHVSockPurpose(p) + if err != nil { + return nil, err + } + + machineName, _, err := k.GetStringValue(HvsockMachineName) + if err != nil { + return nil, err + } + return &HVSockRegistryEntry{ + KeyName: keyName, + Purpose: purpose, + Port: port, + MachineName: machineName, + Key: k, + }, nil +} + +// Listen s used on the windows side to listen for anything to come +// over the hvsock as a signal the vm is booted +func (hv *HVSockRegistryEntry) Listen() error { + n := winio.HvsockAddr{ + VMID: winio.HvsockGUIDWildcard(), // When listening on the host side, use equiv of 0.0.0.0 + ServiceID: winio.VsockServiceID(uint32(hv.Port)), + } + listener, err := winio.ListenHvsock(&n) + if err != nil { + return err + } + defer func() { + if err := listener.Close(); err != nil { + logrus.Error(err) + } + }() + conn, err := listener.Accept() + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + logrus.Error(err) + } + }() + // Right now we just listen for anything down the pipe (like qemu) + _, err = bufio.NewReader(conn).ReadString('\n') + return err +} diff --git a/pkg/machine/ignition.go b/pkg/machine/ignition.go index acee37db41..bda2a68263 100644 --- a/pkg/machine/ignition.go +++ b/pkg/machine/ignition.go @@ -34,12 +34,12 @@ func intToPtr(i int) *int { } // Convenience function to convert string to ptr -func strToPtr(s string) *string { +func StrToPtr(s string) *string { return &s } // Convenience function to convert bool to ptr -func boolToPtr(b bool) *bool { +func BoolToPtr(b bool) *bool { return &b } @@ -57,11 +57,21 @@ type DynamicIgnition struct { TimeZone string UID int VMName string + VMType VMType WritePath string + Cfg Config } -// NewIgnitionFile -func NewIgnitionFile(ign DynamicIgnition, vmType VMType) error { +func (ign *DynamicIgnition) Write() error { + b, err := json.Marshal(ign.Cfg) + if err != nil { + return err + } + return os.WriteFile(ign.WritePath, b, 0644) +} + +// GenerateIgnitionConfig +func (ign *DynamicIgnition) GenerateIgnitionConfig() error { if len(ign.Name) < 1 { ign.Name = DefaultIgnitionUserName } @@ -109,32 +119,17 @@ func NewIgnitionFile(ign DynamicIgnition, vmType VMType) error { Node: Node{ Group: getNodeGrp("root"), Path: "/etc/localtime", - Overwrite: boolToPtr(false), + Overwrite: BoolToPtr(false), User: getNodeUsr("root"), }, LinkEmbedded1: LinkEmbedded1{ - Hard: boolToPtr(false), + Hard: BoolToPtr(false), Target: filepath.Join("/usr/share/zoneinfo", tz), }, } ignStorage.Links = append(ignStorage.Links, tzLink) } - // ready is a unit file that sets up the virtual serial device - // where when the VM is done configuring, it will send an ack - // so a listening host knows it can being interacting with it - ready := `[Unit] -Requires=dev-virtio\\x2dports-%s.device -After=remove-moby.service sshd.socket sshd.service -OnFailure=emergency.target -OnFailureJobMode=isolate -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo Ready >/dev/%s' -[Install] -RequiredBy=default.target -` deMoby := `[Unit] Description=Remove moby-engine # Run once for the machine @@ -190,26 +185,21 @@ WantedBy=sysinit.target ignSystemd := Systemd{ Units: []Unit{ { - Enabled: boolToPtr(true), + Enabled: BoolToPtr(true), Name: "podman.socket", }, { - Enabled: boolToPtr(true), - Name: "ready.service", - Contents: strToPtr(fmt.Sprintf(ready, "vport1p1", "vport1p1")), - }, - { - Enabled: boolToPtr(false), + Enabled: BoolToPtr(false), Name: "docker.service", - Mask: boolToPtr(true), + Mask: BoolToPtr(true), }, { - Enabled: boolToPtr(false), + Enabled: BoolToPtr(false), Name: "docker.socket", - Mask: boolToPtr(true), + Mask: BoolToPtr(true), }, { - Enabled: boolToPtr(true), + Enabled: BoolToPtr(true), Name: "remove-moby.service", Contents: &deMoby, }, @@ -222,20 +212,16 @@ WantedBy=sysinit.target } // Only qemu has the qemu firmware environment setting - if vmType == QemuVirt { + if ign.VMType == QemuVirt { qemuUnit := Unit{ - Enabled: boolToPtr(true), + Enabled: BoolToPtr(true), Name: "envset-fwcfg.service", Contents: &envset, } ignSystemd.Units = append(ignSystemd.Units, qemuUnit) } - - b, err := json.Marshal(ignConfig) - if err != nil { - return err - } - return os.WriteFile(ign.WritePath, b, 0644) + ign.Cfg = ignConfig + return nil } func getDirs(usrName string) []Directory { @@ -363,7 +349,7 @@ Delegate=memory pids cpu io Group: getNodeGrp("root"), Path: sub, User: getNodeUsr("root"), - Overwrite: boolToPtr(true), + Overwrite: BoolToPtr(true), }, FileEmbedded1: FileEmbedded1{ Append: nil, @@ -428,6 +414,7 @@ Delegate=memory pids cpu io FileEmbedded1: FileEmbedded1{ Append: nil, Contents: Resource{ + // TODO this should be fixed for all vmtypes Source: encodeDataURLPtr("qemu\n"), }, Mode: intToPtr(0644), @@ -633,23 +620,23 @@ func getLinks(usrName string) []Link { User: getNodeUsr(usrName), }, LinkEmbedded1: LinkEmbedded1{ - Hard: boolToPtr(false), + Hard: BoolToPtr(false), Target: "/home/" + usrName + "/.config/systemd/user/linger-example.service", }, }, { Node: Node{ Group: getNodeGrp("root"), Path: "/usr/local/bin/docker", - Overwrite: boolToPtr(true), + Overwrite: BoolToPtr(true), User: getNodeUsr("root"), }, LinkEmbedded1: LinkEmbedded1{ - Hard: boolToPtr(false), + Hard: BoolToPtr(false), Target: "/usr/bin/podman", }, }} } func encodeDataURLPtr(contents string) *string { - return strToPtr(fmt.Sprintf("data:,%s", url.PathEscape(contents))) + return StrToPtr(fmt.Sprintf("data:,%s", url.PathEscape(contents))) } diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index f39cbd8abc..38f5fb4808 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -384,16 +384,43 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) { return false, os.WriteFile(v.getIgnitionFile(), inputIgnition, 0644) } // Write the ignition file - ign := machine.DynamicIgnition{ + ign := &machine.DynamicIgnition{ Name: opts.Username, Key: key, VMName: v.Name, + VMType: machine.QemuVirt, TimeZone: opts.TimeZone, WritePath: v.getIgnitionFile(), UID: v.UID, } - err = machine.NewIgnitionFile(ign, machine.QemuVirt) + if err := ign.GenerateIgnitionConfig(); err != nil { + return false, err + } + + // ready is a unit file that sets up the virtual serial device + // where when the VM is done configuring, it will send an ack + // so a listening host knows it can being interacting with it + ready := `[Unit] +Requires=dev-virtio\\x2dports-%s.device +After=remove-moby.service sshd.socket sshd.service +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo Ready >/dev/%s' +[Install] +RequiredBy=default.target +` + readyUnit := machine.Unit{ + Enabled: machine.BoolToPtr(true), + Name: "ready.service", + Contents: machine.StrToPtr(fmt.Sprintf(ready, "vport1p1", "vport1p1")), + } + ign.Cfg.Systemd.Units = append(ign.Cfg.Systemd.Units, readyUnit) + + err = ign.Write() return err == nil, err }