diff --git a/go.mod b/go.mod index 7e256ac256..64c782b74d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( github.com/BurntSushi/toml v1.2.1 + github.com/Microsoft/go-winio v0.6.0 github.com/blang/semver/v4 v4.0.0 github.com/buger/goterm v1.0.4 github.com/checkpoint-restore/checkpointctl v0.1.0 @@ -74,7 +75,6 @@ require ( require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect - github.com/Microsoft/go-winio v0.6.0 // indirect github.com/Microsoft/hcsshim v0.10.0-rc.7 // indirect github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect diff --git a/pkg/machine/hyperv/config.go b/pkg/machine/hyperv/config.go index 04430e545c..8d334a4deb 100644 --- a/pkg/machine/hyperv/config.go +++ b/pkg/machine/hyperv/config.go @@ -6,7 +6,6 @@ package hyperv import ( "encoding/json" "errors" - "github.com/sirupsen/logrus" "io/fs" "os" "path/filepath" @@ -15,6 +14,7 @@ import ( "github.com/containers/libhvee/pkg/hypervctl" "github.com/containers/podman/v4/pkg/machine" "github.com/docker/go-units" + "github.com/sirupsen/logrus" ) type Virtualization struct { @@ -178,7 +178,6 @@ func (v Virtualization) NewMachine(opts machine.InitOptions) (machine.VM, error) return nil, err } return v.LoadVMByName(opts.Name) - } func (v Virtualization) RemoveAndCleanMachines() error { @@ -221,6 +220,12 @@ func (v Virtualization) RemoveAndCleanMachines() error { if err := vm.Remove(mm.ImagePath.GetPath()); err != nil { prevErr = handlePrevError(err, prevErr) } + if err := mm.ReadyHVSock.Remove(); err != nil { + prevErr = handlePrevError(err, prevErr) + } + if err := mm.NetworkHVSock.Remove(); err != nil { + prevErr = handlePrevError(err, prevErr) + } } // Nuke the config and dataDirs diff --git a/pkg/machine/hyperv/machine.go b/pkg/machine/hyperv/machine.go index 177dc8c8a1..06e4388137 100644 --- a/pkg/machine/hyperv/machine.go +++ b/pkg/machine/hyperv/machine.go @@ -56,12 +56,8 @@ const ( ) type HyperVMachine struct { - // copied from qemu, cull and add as needed - // ConfigPath is the fully qualified path to the configuration file ConfigPath machine.VMFile - // The command line representation of the qemu command - //CmdLine []string // HostUser contains info about host user machine.HostUser // ImageConfig describes the bootable image @@ -70,14 +66,10 @@ type HyperVMachine struct { Mounts []machine.Mount // Name of VM Name string - // PidFilePath is the where the Proxy PID file lives - //PidFilePath machine.VMFile - // VMPidFilePath is the where the VM PID file lives - //VMPidFilePath machine.VMFile - // QMPMonitor is the qemu monitor object for sending commands - //QMPMonitor Monitor + // NetworkVSock is for the user networking + NetworkHVSock HVSockRegistryEntry // ReadySocket tells host when vm is booted - ReadySocket machine.VMFile + ReadyHVSock HVSockRegistryEntry // ResourceConfig is physical attrs of the VM machine.ResourceConfig // SSHConfig for accessing the remote vm @@ -95,6 +87,18 @@ func (m *HyperVMachine) Init(opts machine.InitOptions) (bool, error) { key string ) + // Add the network and ready sockets to the Windows registry + networkHVSock, err := NewHVSockRegistryEntry(m.Name, Network) + if err != nil { + return false, err + } + eventHVSocket, err := NewHVSockRegistryEntry(m.Name, Events) + if err != nil { + return false, err + } + m.NetworkHVSock = *networkHVSock + m.ReadyHVSock = *eventHVSocket + sshDir := filepath.Join(homedir.Get(), ".ssh") m.IdentityPath = filepath.Join(sshDir, m.Name) @@ -170,12 +174,39 @@ func (m *HyperVMachine) Init(opts machine.InitOptions) (bool, error) { Name: user, Key: key, VMName: m.Name, + VMType: machine.HyperVVirt, TimeZone: opts.TimeZone, WritePath: m.IgnitionFile.GetPath(), UID: m.UID, } - if err := machine.NewIgnitionFile(ign, machine.HyperVVirt); err != nil { + if err := ign.GenerateIgnitionConfig(); err != nil { + return false, err + } + + // ready is a unit file that sets up the virtual serial device + // where when the VM is done configuring, it will send an ack + // so a listening host knows it can being interacting with it + // + // VSOCK-CONNECT:2 <- shortcut to connect to the hostvm + ready := `[Unit] +After=remove-moby.service sshd.socket sshd.service +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo Ready | socat - VSOCK-CONNECT:2:%d' +[Install] +RequiredBy=default.target +` + readyUnit := machine.Unit{ + Enabled: machine.BoolToPtr(true), + Name: "ready.service", + Contents: machine.StrToPtr(fmt.Sprintf(ready, m.ReadyHVSock.Port)), + } + ign.Cfg.Systemd.Units = append(ign.Cfg.Systemd.Units, readyUnit) + if err := ign.Write(); err != nil { return false, err } // The ignition file has been written. We now need to @@ -259,12 +290,6 @@ func (m *HyperVMachine) Remove(_ string, opts machine.RemoveOptions) (string, fu files = append(files, diskPath) } - if err := machine.RemoveConnection(m.Name); err != nil { - logrus.Error(err) - } - if err := machine.RemoveConnection(m.Name + "-root"); err != nil { - logrus.Error(err) - } files = append(files, getVMConfigPath(m.ConfigPath.GetPath(), m.Name)) confirmationMessage := "\nThe following files will be deleted:\n\n" for _, msg := range files { @@ -278,6 +303,22 @@ func (m *HyperVMachine) Remove(_ string, opts machine.RemoveOptions) (string, fu logrus.Error(err) } } + if err := machine.RemoveConnection(m.Name); err != nil { + logrus.Error(err) + } + if err := machine.RemoveConnection(m.Name + "-root"); err != nil { + logrus.Error(err) + } + + // Remove the HVSOCK for networking + if err := m.NetworkHVSock.Remove(); err != nil { + logrus.Errorf("unable to remove registry entry for %s: %q", m.NetworkHVSock.KeyName, err) + } + + // Remove the HVSOCK for events + if err := m.ReadyHVSock.Remove(); err != nil { + logrus.Errorf("unable to remove registry entry for %s: %q", m.NetworkHVSock.KeyName, err) + } return vm.Remove(diskPath) }, nil } @@ -360,7 +401,11 @@ func (m *HyperVMachine) Start(name string, opts machine.StartOptions) error { if vm.State() != hypervctl.Disabled { return hypervctl.ErrMachineStateInvalid } - return vm.Start() + if err := vm.Start(); err != nil { + return err + } + // Wait on notification from the guest + return m.ReadyHVSock.Listen() } func (m *HyperVMachine) State(_ bool) (machine.Status, error) { diff --git a/pkg/machine/hyperv/vsock.go b/pkg/machine/hyperv/vsock.go new file mode 100644 index 0000000000..f4bcbc9358 --- /dev/null +++ b/pkg/machine/hyperv/vsock.go @@ -0,0 +1,270 @@ +//go:build windows +// +build windows + +package hyperv + +import ( + "bufio" + "errors" + "fmt" + "strings" + + "github.com/Microsoft/go-winio" + "github.com/containers/podman/v4/utils" + "github.com/sirupsen/logrus" + "golang.org/x/sys/windows/registry" +) + +var ErrVSockRegistryEntryExists = errors.New("registry entry already exists") + +const ( + // HvsockMachineName is the string identifier for the machine name in a registry entry + HvsockMachineName = "MachineName" + // HvsockPurpose is the string identifier for the sock purpose in a registry entry + HvsockPurpose = "Purpose" + // VsockRegistryPath describes the registry path to where the hvsock registry entries live + VsockRegistryPath = `SOFTWARE\Microsoft\Windows NT\CurrentVersion\Virtualization\GuestCommunicationServices` + // LinuxVm is the default guid for a Linux VM on Windows + LinuxVm = "FACB-11E6-BD58-64006A7986D3" +) + +// HVSockPurpose describes what the hvsock is needed for +type HVSockPurpose int + +const ( + // Network implies the sock is used for user-mode networking + Network HVSockPurpose = iota + // Events implies the sock is used for notification (like "Ready") + Events +) + +func (hv HVSockPurpose) string() string { + switch hv { + case Network: + return "Network" + case Events: + return "Events" + } + return "" +} + +func (hv HVSockPurpose) Equal(purpose string) bool { + return hv.string() == purpose +} + +func toHVSockPurpose(p string) (HVSockPurpose, error) { + switch p { + case "Network": + return Network, nil + case "Events": + return Events, nil + } + return 0, fmt.Errorf("unknown hvsockpurpose: %s", p) +} + +func openVSockRegistryEntry(entry string) (registry.Key, error) { + return registry.OpenKey(registry.LOCAL_MACHINE, entry, registry.QUERY_VALUE) +} + +// HVSockRegistryEntry describes a registry entry used in Windows for HVSOCK implementations +type HVSockRegistryEntry struct { + KeyName string `json:"key_name"` + Purpose HVSockPurpose `json:"purpose"` + Port uint64 `json:"port"` + MachineName string `json:"machineName"` + Key registry.Key `json:"key,omitempty"` +} + +// Add creates a new Windows registry entry with string values from the +// HVSockRegistryEntry. +func (hv *HVSockRegistryEntry) Add() error { + if err := hv.validate(); err != nil { + return err + } + exists, err := hv.exists() + if err != nil { + return err + } + if exists { + return fmt.Errorf("%q: %s", ErrVSockRegistryEntryExists, hv.KeyName) + } + parentKey, err := registry.OpenKey(registry.LOCAL_MACHINE, VsockRegistryPath, registry.QUERY_VALUE) + defer func() { + if err := parentKey.Close(); err != nil { + logrus.Error(err) + } + }() + if err != nil { + return err + } + newKey, _, err := registry.CreateKey(parentKey, hv.KeyName, registry.WRITE) + defer func() { + if err := newKey.Close(); err != nil { + logrus.Error(err) + } + }() + if err != nil { + return err + } + + if err := newKey.SetStringValue(HvsockPurpose, hv.Purpose.string()); err != nil { + return err + } + return newKey.SetStringValue(HvsockMachineName, hv.MachineName) +} + +// Remove deletes the registry key and its string values +func (hv *HVSockRegistryEntry) Remove() error { + return registry.DeleteKey(registry.LOCAL_MACHINE, hv.fqPath()) +} + +func (hv *HVSockRegistryEntry) fqPath() string { + return fmt.Sprintf("%s\\%s", VsockRegistryPath, hv.KeyName) +} + +func (hv *HVSockRegistryEntry) validate() error { + if hv.Port < 1 { + return errors.New("port must be larger than 1") + } + if len(hv.Purpose.string()) < 1 { + return errors.New("required field purpose is empty") + } + if len(hv.MachineName) < 1 { + return errors.New("required field machinename is empty") + } + if len(hv.KeyName) < 1 { + return errors.New("required field keypath is empty") + } + //decimal_num, err = strconv.ParseInt(hexadecimal_num, 16, 64) + return nil +} + +func (hv *HVSockRegistryEntry) exists() (bool, error) { + foo := hv.fqPath() + _ = foo + _, err := openVSockRegistryEntry(hv.fqPath()) + if err == nil { + return true, err + } + if errors.Is(err, registry.ErrNotExist) { + return false, nil + } + return false, err +} + +// findOpenHVSockPort looks for an open random port. it verifies the port is not +// already being used by another hvsock in the Windows registry. +func findOpenHVSockPort() (uint64, error) { + // If we cannot find a free port in 10 attempts, something is wrong + for i := 0; i < 10; i++ { + port, err := utils.GetRandomPort() + if err != nil { + return 0, err + } + // Try and load registry entries by port to see if they exist + _, err = LoadHVSockRegistryEntry(uint64(port)) + if err == nil { + // the port is no good, it is being used; try again + logrus.Errorf("port %d is already used for hvsock", port) + continue + } + if errors.Is(err, registry.ErrNotExist) { + // the port is good to go + return uint64(port), nil + } + if err != nil { + // something went wrong + return 0, err + } + } + return 0, errors.New("unable to find a free port for hvsock use") +} + +// NewHVSockRegistryEntry is a constructor to make a new registry entry in Windows. After making the new +// object, you must call the add() method to *actually* add it to the Windows registry. +func NewHVSockRegistryEntry(machineName string, purpose HVSockPurpose) (*HVSockRegistryEntry, error) { + // a so-called wildcard entry ... everything from FACB -> 6D3 is MS special sauce + // for a " linux vm". this first segment is hexi for the hvsock port number + //00000400-FACB-11E6-BD58-64006A7986D3 + port, err := findOpenHVSockPort() + if err != nil { + return nil, err + } + r := HVSockRegistryEntry{ + KeyName: portToKeyName(port), + Purpose: purpose, + Port: port, + MachineName: machineName, + } + if err := r.Add(); err != nil { + return nil, err + } + return &r, nil +} + +func portToKeyName(port uint64) string { + // this could be flattened but given the complexity, I thought it might + // be more difficult to read + hexi := strings.ToUpper(fmt.Sprintf("%08x", port)) + return fmt.Sprintf("%s-%s", hexi, LinuxVm) +} + +func LoadHVSockRegistryEntry(port uint64) (*HVSockRegistryEntry, error) { + keyName := portToKeyName(port) + fqPath := fmt.Sprintf("%s\\%s", VsockRegistryPath, keyName) + k, err := openVSockRegistryEntry(fqPath) + if err != nil { + return nil, err + } + p, _, err := k.GetStringValue(HvsockPurpose) + if err != nil { + return nil, err + } + + purpose, err := toHVSockPurpose(p) + if err != nil { + return nil, err + } + + machineName, _, err := k.GetStringValue(HvsockMachineName) + if err != nil { + return nil, err + } + return &HVSockRegistryEntry{ + KeyName: keyName, + Purpose: purpose, + Port: port, + MachineName: machineName, + Key: k, + }, nil +} + +// Listen s used on the windows side to listen for anything to come +// over the hvsock as a signal the vm is booted +func (hv *HVSockRegistryEntry) Listen() error { + n := winio.HvsockAddr{ + VMID: winio.HvsockGUIDWildcard(), // When listening on the host side, use equiv of 0.0.0.0 + ServiceID: winio.VsockServiceID(uint32(hv.Port)), + } + listener, err := winio.ListenHvsock(&n) + if err != nil { + return err + } + defer func() { + if err := listener.Close(); err != nil { + logrus.Error(err) + } + }() + conn, err := listener.Accept() + if err != nil { + return err + } + defer func() { + if err := conn.Close(); err != nil { + logrus.Error(err) + } + }() + // Right now we just listen for anything down the pipe (like qemu) + _, err = bufio.NewReader(conn).ReadString('\n') + return err +} diff --git a/pkg/machine/ignition.go b/pkg/machine/ignition.go index acee37db41..bda2a68263 100644 --- a/pkg/machine/ignition.go +++ b/pkg/machine/ignition.go @@ -34,12 +34,12 @@ func intToPtr(i int) *int { } // Convenience function to convert string to ptr -func strToPtr(s string) *string { +func StrToPtr(s string) *string { return &s } // Convenience function to convert bool to ptr -func boolToPtr(b bool) *bool { +func BoolToPtr(b bool) *bool { return &b } @@ -57,11 +57,21 @@ type DynamicIgnition struct { TimeZone string UID int VMName string + VMType VMType WritePath string + Cfg Config } -// NewIgnitionFile -func NewIgnitionFile(ign DynamicIgnition, vmType VMType) error { +func (ign *DynamicIgnition) Write() error { + b, err := json.Marshal(ign.Cfg) + if err != nil { + return err + } + return os.WriteFile(ign.WritePath, b, 0644) +} + +// GenerateIgnitionConfig +func (ign *DynamicIgnition) GenerateIgnitionConfig() error { if len(ign.Name) < 1 { ign.Name = DefaultIgnitionUserName } @@ -109,32 +119,17 @@ func NewIgnitionFile(ign DynamicIgnition, vmType VMType) error { Node: Node{ Group: getNodeGrp("root"), Path: "/etc/localtime", - Overwrite: boolToPtr(false), + Overwrite: BoolToPtr(false), User: getNodeUsr("root"), }, LinkEmbedded1: LinkEmbedded1{ - Hard: boolToPtr(false), + Hard: BoolToPtr(false), Target: filepath.Join("/usr/share/zoneinfo", tz), }, } ignStorage.Links = append(ignStorage.Links, tzLink) } - // ready is a unit file that sets up the virtual serial device - // where when the VM is done configuring, it will send an ack - // so a listening host knows it can being interacting with it - ready := `[Unit] -Requires=dev-virtio\\x2dports-%s.device -After=remove-moby.service sshd.socket sshd.service -OnFailure=emergency.target -OnFailureJobMode=isolate -[Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/bin/sh -c '/usr/bin/echo Ready >/dev/%s' -[Install] -RequiredBy=default.target -` deMoby := `[Unit] Description=Remove moby-engine # Run once for the machine @@ -190,26 +185,21 @@ WantedBy=sysinit.target ignSystemd := Systemd{ Units: []Unit{ { - Enabled: boolToPtr(true), + Enabled: BoolToPtr(true), Name: "podman.socket", }, { - Enabled: boolToPtr(true), - Name: "ready.service", - Contents: strToPtr(fmt.Sprintf(ready, "vport1p1", "vport1p1")), - }, - { - Enabled: boolToPtr(false), + Enabled: BoolToPtr(false), Name: "docker.service", - Mask: boolToPtr(true), + Mask: BoolToPtr(true), }, { - Enabled: boolToPtr(false), + Enabled: BoolToPtr(false), Name: "docker.socket", - Mask: boolToPtr(true), + Mask: BoolToPtr(true), }, { - Enabled: boolToPtr(true), + Enabled: BoolToPtr(true), Name: "remove-moby.service", Contents: &deMoby, }, @@ -222,20 +212,16 @@ WantedBy=sysinit.target } // Only qemu has the qemu firmware environment setting - if vmType == QemuVirt { + if ign.VMType == QemuVirt { qemuUnit := Unit{ - Enabled: boolToPtr(true), + Enabled: BoolToPtr(true), Name: "envset-fwcfg.service", Contents: &envset, } ignSystemd.Units = append(ignSystemd.Units, qemuUnit) } - - b, err := json.Marshal(ignConfig) - if err != nil { - return err - } - return os.WriteFile(ign.WritePath, b, 0644) + ign.Cfg = ignConfig + return nil } func getDirs(usrName string) []Directory { @@ -363,7 +349,7 @@ Delegate=memory pids cpu io Group: getNodeGrp("root"), Path: sub, User: getNodeUsr("root"), - Overwrite: boolToPtr(true), + Overwrite: BoolToPtr(true), }, FileEmbedded1: FileEmbedded1{ Append: nil, @@ -428,6 +414,7 @@ Delegate=memory pids cpu io FileEmbedded1: FileEmbedded1{ Append: nil, Contents: Resource{ + // TODO this should be fixed for all vmtypes Source: encodeDataURLPtr("qemu\n"), }, Mode: intToPtr(0644), @@ -633,23 +620,23 @@ func getLinks(usrName string) []Link { User: getNodeUsr(usrName), }, LinkEmbedded1: LinkEmbedded1{ - Hard: boolToPtr(false), + Hard: BoolToPtr(false), Target: "/home/" + usrName + "/.config/systemd/user/linger-example.service", }, }, { Node: Node{ Group: getNodeGrp("root"), Path: "/usr/local/bin/docker", - Overwrite: boolToPtr(true), + Overwrite: BoolToPtr(true), User: getNodeUsr("root"), }, LinkEmbedded1: LinkEmbedded1{ - Hard: boolToPtr(false), + Hard: BoolToPtr(false), Target: "/usr/bin/podman", }, }} } func encodeDataURLPtr(contents string) *string { - return strToPtr(fmt.Sprintf("data:,%s", url.PathEscape(contents))) + return StrToPtr(fmt.Sprintf("data:,%s", url.PathEscape(contents))) } diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index f39cbd8abc..38f5fb4808 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -384,16 +384,43 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) { return false, os.WriteFile(v.getIgnitionFile(), inputIgnition, 0644) } // Write the ignition file - ign := machine.DynamicIgnition{ + ign := &machine.DynamicIgnition{ Name: opts.Username, Key: key, VMName: v.Name, + VMType: machine.QemuVirt, TimeZone: opts.TimeZone, WritePath: v.getIgnitionFile(), UID: v.UID, } - err = machine.NewIgnitionFile(ign, machine.QemuVirt) + if err := ign.GenerateIgnitionConfig(); err != nil { + return false, err + } + + // ready is a unit file that sets up the virtual serial device + // where when the VM is done configuring, it will send an ack + // so a listening host knows it can being interacting with it + ready := `[Unit] +Requires=dev-virtio\\x2dports-%s.device +After=remove-moby.service sshd.socket sshd.service +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c '/usr/bin/echo Ready >/dev/%s' +[Install] +RequiredBy=default.target +` + readyUnit := machine.Unit{ + Enabled: machine.BoolToPtr(true), + Name: "ready.service", + Contents: machine.StrToPtr(fmt.Sprintf(ready, "vport1p1", "vport1p1")), + } + ign.Cfg.Systemd.Units = append(ign.Cfg.Systemd.Units, readyUnit) + + err = ign.Write() return err == nil, err }