From 91ef6869da040340548ce14ec9f7b09887553065 Mon Sep 17 00:00:00 2001 From: Christophe Fergeau Date: Wed, 26 Jul 2023 17:22:21 +0200 Subject: [PATCH] Add rosetta support This adds support for `--device rosetta,mountTag=something` on the commandline. This is only available on system with Apple CPUs, vfkit will error out if this option is used on Intel CPUs. Once the VM is running and the rosetta share is mounted, rosetta support can be enabled by creating this file: $ cat /etc/binfmt.d/rosetta.conf :rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/rosetta:F and then running `systemctl restart systemd-binfmt` See these links for more details: https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta?language=objc https://docs.kernel.org/admin-guide/binfmt-misc.html https://www.man7.org/linux/man-pages/man5/binfmt.d.5.html This fixes https://github.com/crc-org/vfkit/issues/23 --- doc/usage.md | 32 ++++++++++++++++++++ pkg/config/json.go | 16 ++++++++++ pkg/config/json_test.go | 7 ++++- pkg/config/virtio.go | 58 +++++++++++++++++++++++++++++++++++-- pkg/config/virtio_test.go | 13 ++++++++- pkg/vf/rosetta_amd64.go | 9 ++++++ pkg/vf/rosetta_arm64.go | 61 +++++++++++++++++++++++++++++++++++++++ pkg/vf/virtio.go | 3 ++ 8 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 pkg/vf/rosetta_amd64.go create mode 100644 pkg/vf/rosetta_arm64.go diff --git a/doc/usage.md b/doc/usage.md index 573ff6d1..c03bf467 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -214,6 +214,38 @@ The share can be mounted in the guest with `mount -t virtiofs vfkitTag /mnt`, wi #### Example `--device virtio-fs,sharedDir=/Users/virtuser/vfkit/,mountTag=vfkit-share` +### Rosetta + +#### Description + +The `-device rosetta` option allows to use Rosetta to run x86_64 binaries in an arm64 linux VM. This option will share a directory containing the rosetta binaries over virtio-fs. +The share can be mounted in the guest with `mount -t virtiofs vfkitTag /mnt`, with `vfkitTag` corresponding to the value of the `mountTag` option. +Then, [`binfmt`](https://docs.kernel.org/admin-guide/binfmt-misc.html) needs to be configured to use this rosetta binary for x86_64 executables. +On systems using systemd, this can be achieved by creating a /etc/binfmt.d/rosetta.conf file with this content (`/mnt/rosetta` is the full path to the rosetta binary): +``` +:rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/rosetta:F +``` +and then running `systemctl restart systemd-binfmt`. + +This option is only available on machine with Apple CPUs, `vfkit` will fail with an error if it's used on Intel machines. + +See https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta?language=objc for more details. + + +#### Arguments +- `mountTag`: tag which will be used to mount the rosetta share in the guest. +- `install`: indicates to automatically install rosetta on systems where it's missing. By default, an error will be reported if `--device rosetta` is used when rosetta is not installed. + +#### Example + +This adds rosetta support to the guest: +``` +--device rosetta,mountTag=rosetta-share +``` + +The share can then be mounted with `mount -t virtiofs rosetta-share /mnt`. + + ### GPU #### Description diff --git a/pkg/config/json.go b/pkg/config/json.go index 15ba5bc5..8d7be706 100644 --- a/pkg/config/json.go +++ b/pkg/config/json.go @@ -25,6 +25,7 @@ const ( vfGpu vmComponentKind = "virtiogpu" vfInput vmComponentKind = "virtioinput" usbMassStorage vmComponentKind = "usbmassstorage" + rosetta vmComponentKind = "rosetta" ) type jsonKind struct { @@ -112,6 +113,10 @@ func unmarshalDevice(rawMsg json.RawMessage) (VirtioDevice, error) { var newDevice VirtioFs err = json.Unmarshal(rawMsg, &newDevice) dev = &newDevice + case rosetta: + var newDevice RosettaShare + err = json.Unmarshal(rawMsg, &newDevice) + dev = &newDevice case vfRng: var newDevice VirtioRng err = json.Unmarshal(rawMsg, &newDevice) @@ -253,6 +258,17 @@ func (dev *VirtioFs) MarshalJSON() ([]byte, error) { }) } +func (dev *RosettaShare) MarshalJSON() ([]byte, error) { + type devWithKind struct { + jsonKind + RosettaShare + } + return json.Marshal(devWithKind{ + jsonKind: kind(rosetta), + RosettaShare: *dev, + }) +} + func (dev *VirtioRng) MarshalJSON() ([]byte, error) { type devWithKind struct { jsonKind diff --git a/pkg/config/json_test.go b/pkg/config/json_test.go index 6acddb9c..0c2184f3 100644 --- a/pkg/config/json_test.go +++ b/pkg/config/json_test.go @@ -106,10 +106,15 @@ var jsonTests = map[string]jsonTest{ require.NoError(t, err) err = vm.AddDevice(dev) require.NoError(t, err) + // rosetta + dev, err = RosettaShareNew("vz-rosetta") + require.NoError(t, err) + err = vm.AddDevice(dev) + require.NoError(t, err) return vm }, - expectedJSON: `{"vcpus":3,"memoryBytes":4000000000,"bootloader":{"kind":"linuxBootloader","VmlinuzPath":"/vmlinuz","KernelCmdLine":"/initrd","InitrdPath":"console=hvc0"},"devices":[{"kind":"virtioserial","LogFile":"/virtioserial","UsesStdio":false},{"kind":"virtioinput","inputType":"keyboard"},{"kind":"virtiogpu","usesGUI":false,"width":800,"height":600},{"kind":"virtionet","Nat":true,"MacAddress":"ABEiM0RV","Socket":null,"UnixSocketPath":""},{"kind":"virtiorng"},{"kind":"virtioblk","DevName":"virtio-blk","ImagePath":"/virtioblk","ReadOnly":false,"DeviceIdentifier":""},{"kind":"virtiosock","Port":1234,"SocketURL":"/virtiovsock","Listen":false},{"kind":"virtiofs","SharedDir":"/virtiofs","MountTag":"tag"},{"kind":"usbmassstorage","DevName":"usb-mass-storage","ImagePath":"/usbmassstorage","ReadOnly":false}]}`, + expectedJSON: `{"vcpus":3,"memoryBytes":4000000000,"bootloader":{"kind":"linuxBootloader","VmlinuzPath":"/vmlinuz","KernelCmdLine":"/initrd","InitrdPath":"console=hvc0"},"devices":[{"kind":"virtioserial","LogFile":"/virtioserial","UsesStdio":false},{"kind":"virtioinput","inputType":"keyboard"},{"kind":"virtiogpu","usesGUI":false,"width":800,"height":600},{"kind":"virtionet","Nat":true,"MacAddress":"ABEiM0RV","Socket":null,"UnixSocketPath":""},{"kind":"virtiorng"},{"kind":"virtioblk","DevName":"virtio-blk","ImagePath":"/virtioblk","ReadOnly":false,"DeviceIdentifier":""},{"kind":"virtiosock","Port":1234,"SocketURL":"/virtiovsock","Listen":false},{"kind":"virtiofs","MountTag":"tag","SharedDir":"/virtiofs"},{"kind":"usbmassstorage","DevName":"usb-mass-storage","ImagePath":"/usbmassstorage","ReadOnly":false},{"kind":"rosetta","MountTag":"vz-rosetta","InstallRosetta":false}]}`, }, } diff --git a/pkg/config/virtio.go b/pkg/config/virtio.go index 7d7ba63f..318e08d7 100644 --- a/pkg/config/virtio.go +++ b/pkg/config/virtio.go @@ -61,10 +61,20 @@ type VirtioBlk struct { DeviceIdentifier string } +type DirectorySharingConfig struct { + MountTag string +} + // VirtioFs configures directory sharing between the guest and the host. type VirtioFs struct { + DirectorySharingConfig SharedDir string - MountTag string +} + +// RosettaShare configures rosetta support in the guest to run Intel binaries on Apple CPUs +type RosettaShare struct { + DirectorySharingConfig + InstallRosetta bool } // virtioRng configures a random number generator (RNG) device. @@ -131,6 +141,8 @@ func deviceFromCmdLine(deviceOpts string) (VirtioDevice, error) { } var dev VirtioDevice switch opts[0] { + case "rosetta": + dev = &RosettaShare{} case "virtio-blk": dev = virtioBlkNewEmpty() case "virtio-fs": @@ -545,8 +557,10 @@ func (dev *VirtioVsock) FromOptions(options []option) error { // mounted in the VM using `mount -t virtiofs mountTag /some/dir` func VirtioFsNew(sharedDir string, mountTag string) (VirtioDevice, error) { return &VirtioFs{ + DirectorySharingConfig: DirectorySharingConfig{ + MountTag: mountTag, + }, SharedDir: sharedDir, - MountTag: mountTag, }, nil } @@ -575,6 +589,46 @@ func (dev *VirtioFs) FromOptions(options []option) error { return nil } +// RosettaShare creates a new rosetta share for running x86_64 binaries on M1 machines. +// It will share a directory containing the linux rosetta binaries with the +// virtual machine. This directory can be mounted in the VM using `mount -t +// virtiofs mountTag /some/dir` +func RosettaShareNew(mountTag string) (VirtioDevice, error) { + return &RosettaShare{ + DirectorySharingConfig: DirectorySharingConfig{ + MountTag: mountTag, + }, + }, nil +} + +func (dev *RosettaShare) ToCmdLine() ([]string, error) { + if dev.MountTag == "" { + return nil, fmt.Errorf("rosetta shares require a mount tag to be specified") + } + builder := strings.Builder{} + builder.WriteString("rosetta") + fmt.Fprintf(&builder, ",mountTag=%s", dev.MountTag) + if dev.InstallRosetta { + builder.WriteString(",install") + } + + return []string{"--device", builder.String()}, nil +} + +func (dev *RosettaShare) FromOptions(options []option) error { + for _, option := range options { + switch option.key { + case "mountTag": + dev.MountTag = option.value + case "install": + dev.InstallRosetta = true + default: + return fmt.Errorf("Unknown option for rosetta share: %s", option.key) + } + } + return nil +} + type USBMassStorage struct { StorageConfig } diff --git a/pkg/config/virtio_test.go b/pkg/config/virtio_test.go index 82b67d54..0b947a18 100644 --- a/pkg/config/virtio_test.go +++ b/pkg/config/virtio_test.go @@ -56,11 +56,22 @@ var virtioDevTests = map[string]virtioDevTest{ newDev: func() (VirtioDevice, error) { return VirtioFsNew("/foo/bar", "myTag") }, expectedDev: &VirtioFs{ SharedDir: "/foo/bar", - MountTag: "myTag", + DirectorySharingConfig: DirectorySharingConfig{ + MountTag: "myTag", + }, }, expectedCmdLine: []string{"--device", "virtio-fs,sharedDir=/foo/bar,mountTag=myTag"}, alternateCmdLine: []string{"--device", "virtio-fs,mountTag=myTag,sharedDir=/foo/bar"}, }, + "NewRosettaShare": { + newDev: func() (VirtioDevice, error) { return RosettaShareNew("myTag") }, + expectedDev: &RosettaShare{ + DirectorySharingConfig: DirectorySharingConfig{ + MountTag: "myTag", + }, + }, + expectedCmdLine: []string{"--device", "rosetta,mountTag=myTag"}, + }, "NewVirtioVsock": { newDev: func() (VirtioDevice, error) { return VirtioVsockNew(1234, "/foo/bar.unix", false) }, expectedDev: &VirtioVsock{ diff --git a/pkg/vf/rosetta_amd64.go b/pkg/vf/rosetta_amd64.go new file mode 100644 index 00000000..ddd3bc01 --- /dev/null +++ b/pkg/vf/rosetta_amd64.go @@ -0,0 +1,9 @@ +package vf + +import ( + "fmt" +) + +func (dev *RosettaShare) AddToVirtualMachineConfig(_ *vzVirtualMachineConfiguration) error { + return fmt.Errorf("rosetta is unsupported on non-arm64 platforms") +} diff --git a/pkg/vf/rosetta_arm64.go b/pkg/vf/rosetta_arm64.go new file mode 100644 index 00000000..9eb4ef5d --- /dev/null +++ b/pkg/vf/rosetta_arm64.go @@ -0,0 +1,61 @@ +package vf + +import ( + "fmt" + + "github.com/Code-Hex/vz/v3" + log "github.com/sirupsen/logrus" +) + +func checkRosettaAvailability(install bool) error { + availability := vz.LinuxRosettaDirectoryShareAvailability() + switch availability { + case vz.LinuxRosettaAvailabilityNotSupported: + return fmt.Errorf("rosetta is not supported") + case vz.LinuxRosettaAvailabilityNotInstalled: + if !install { + return fmt.Errorf("rosetta is not installed") + } + log.Debugf("installing rosetta") + if err := vz.LinuxRosettaDirectoryShareInstallRosetta(); err != nil { + return fmt.Errorf("failed to install rosetta: %w", err) + } + log.Debugf("rosetta installed") + case vz.LinuxRosettaAvailabilityInstalled: + // nothing to do + } + + return nil +} + +func (dev *RosettaShare) toVz() (vz.DirectorySharingDeviceConfiguration, error) { + if dev.MountTag == "" { + return nil, fmt.Errorf("missing mandatory 'mountTage' option for rosetta share") + } + if err := checkRosettaAvailability(dev.InstallRosetta); err != nil { + return nil, err + } + + rosettaShare, err := vz.NewLinuxRosettaDirectoryShare() + if err != nil { + return nil, fmt.Errorf("failed to create a new rosetta directory share: %w", err) + } + config, err := vz.NewVirtioFileSystemDeviceConfiguration(dev.MountTag) + if err != nil { + return nil, fmt.Errorf("failed to create a new virtio file system configuration for rosetta: %w", err) + } + + config.SetDirectoryShare(rosettaShare) + + return config, nil +} + +func (dev *RosettaShare) AddToVirtualMachineConfig(vmConfig *vzVirtualMachineConfiguration) error { + fileSystemDeviceConfig, err := dev.toVz() + if err != nil { + return err + } + log.Infof("Adding virtio-fs device") + vmConfig.directorySharingDevicesConfiguration = append(vmConfig.directorySharingDevicesConfiguration, fileSystemDeviceConfig) + return nil +} diff --git a/pkg/vf/virtio.go b/pkg/vf/virtio.go index ea9f70f1..c6cdd250 100644 --- a/pkg/vf/virtio.go +++ b/pkg/vf/virtio.go @@ -13,6 +13,7 @@ import ( "golang.org/x/sys/unix" ) +type RosettaShare config.RosettaShare type VirtioBlk config.VirtioBlk type VirtioFs config.VirtioFs type VirtioRng config.VirtioRng @@ -248,6 +249,8 @@ func AddToVirtualMachineConfig(dev config.VirtioDevice, vmConfig *vzVirtualMachi return (*USBMassStorage)(d).AddToVirtualMachineConfig(vmConfig) case *config.VirtioBlk: return (*VirtioBlk)(d).AddToVirtualMachineConfig(vmConfig) + case *config.RosettaShare: + return (*RosettaShare)(d).AddToVirtualMachineConfig(vmConfig) case *config.VirtioFs: return (*VirtioFs)(d).AddToVirtualMachineConfig(vmConfig) case *config.VirtioNet: