Skip to content

Commit

Permalink
Add rosetta support
Browse files Browse the repository at this point in the history
This adds support for `--device rosetta,mountTag=something` on the
commandline. This is only available on system with Apple CPUs, vfkit
will error out if this option is used on Intel CPUs.

Once the VM is running and the rosetta share is mounted, rosetta support can be enabled by creating this file:

$ cat /etc/binfmt.d/rosetta.conf
:rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/rosetta:F

and then running `systemctl restart systemd-binfmt`

See these links for more details:
https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta?language=objc
https://docs.kernel.org/admin-guide/binfmt-misc.html
https://www.man7.org/linux/man-pages/man5/binfmt.d.5.html

This fixes crc-org#23
  • Loading branch information
cfergeau committed Jul 28, 2023
1 parent 0eb1914 commit 9a0f904
Show file tree
Hide file tree
Showing 8 changed files with 195 additions and 4 deletions.
32 changes: 32 additions & 0 deletions doc/usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,38 @@ The share can be mounted in the guest with `mount -t virtiofs vfkitTag /mnt`, wi
#### Example
`--device virtio-fs,sharedDir=/Users/virtuser/vfkit/,mountTag=vfkit-share`

### Rosetta

#### Description

The `-device rosetta` option allows to use Rosetta to run x86_64 binaries in an arm64 linux VM. This option will share a directory containing the rosetta binaries over virtio-fs.
The share can be mounted in the guest with `mount -t virtiofs vfkitTag /mnt`, with `vfkitTag` corresponding to the value of the `mountTag` option.
Then, [`binfmt`](https://docs.kernel.org/admin-guide/binfmt-misc.html) needs to be configured to use this rosetta binary for x86_64 executables.
On systems using systemd, this can be achieved by creating a /etc/binfmt.d/rosetta.conf file with this content (`/mnt/rosetta` is the full path to the rosetta binary):
```
:rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/rosetta:F
```
and then running `systemctl restart systemd-binfmt`.

This option is only available on machine with Apple CPUs, `vfkit` will fail with an error if it's used on Intel machines.

See https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta?language=objc for more details.


#### Arguments
- `mountTag`: tag which will be used to mount the rosetta share in the guest.
- `install`: indicates to automatically install rosetta on systems where it's missing. By default, an error will be reported if `--device rosetta` is used when rosetta is not installed.

#### Example

This adds rosetta support to the guest:
```
--device rosetta,mountTag=rosetta-share
```

The share can then be mounted with `mount -t virtiofs rosetta-share /mnt`.


### GPU

#### Description
Expand Down
16 changes: 16 additions & 0 deletions pkg/config/json.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const (
vfGpu vmComponentKind = "virtiogpu"
vfInput vmComponentKind = "virtioinput"
usbMassStorage vmComponentKind = "usbmassstorage"
rosetta vmComponentKind = "rosetta"
)

type jsonKind struct {
Expand Down Expand Up @@ -112,6 +113,10 @@ func unmarshalDevice(rawMsg json.RawMessage) (VirtioDevice, error) {
var newDevice VirtioFs
err = json.Unmarshal(rawMsg, &newDevice)
dev = &newDevice
case rosetta:
var newDevice RosettaShare
err = json.Unmarshal(rawMsg, &newDevice)
dev = &newDevice
case vfRng:
var newDevice VirtioRng
err = json.Unmarshal(rawMsg, &newDevice)
Expand Down Expand Up @@ -253,6 +258,17 @@ func (dev *VirtioFs) MarshalJSON() ([]byte, error) {
})
}

func (dev *RosettaShare) MarshalJSON() ([]byte, error) {
type devWithKind struct {
jsonKind
RosettaShare
}
return json.Marshal(devWithKind{
jsonKind: kind(rosetta),
RosettaShare: *dev,
})
}

func (dev *VirtioRng) MarshalJSON() ([]byte, error) {
type devWithKind struct {
jsonKind
Expand Down
7 changes: 6 additions & 1 deletion pkg/config/json_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,10 +106,15 @@ var jsonTests = map[string]jsonTest{
require.NoError(t, err)
err = vm.AddDevice(dev)
require.NoError(t, err)
// rosetta
dev, err = RosettaShareNew("vz-rosetta")
require.NoError(t, err)
err = vm.AddDevice(dev)
require.NoError(t, err)

return vm
},
expectedJSON: `{"vcpus":3,"memoryBytes":4000000000,"bootloader":{"kind":"linuxBootloader","VmlinuzPath":"/vmlinuz","KernelCmdLine":"/initrd","InitrdPath":"console=hvc0"},"devices":[{"kind":"virtioserial","LogFile":"/virtioserial","UsesStdio":false},{"kind":"virtioinput","inputType":"keyboard"},{"kind":"virtiogpu","usesGUI":false,"height":800,"width":600},{"kind":"virtionet","Nat":true,"MacAddress":"ABEiM0RV","Socket":null,"UnixSocketPath":""},{"kind":"virtiorng"},{"kind":"virtioblk","DevName":"virtio-blk","ImagePath":"/virtioblk","ReadOnly":false,"DeviceIdentifier":""},{"kind":"virtiosock","Port":1234,"SocketURL":"/virtiovsock","Listen":false},{"kind":"virtiofs","SharedDir":"/virtiofs","MountTag":"tag"},{"kind":"usbmassstorage","DevName":"usb-mass-storage","ImagePath":"/usbmassstorage","ReadOnly":false}]}`,
expectedJSON: `{"vcpus":3,"memoryBytes":4000000000,"bootloader":{"kind":"linuxBootloader","VmlinuzPath":"/vmlinuz","KernelCmdLine":"/initrd","InitrdPath":"console=hvc0"},"devices":[{"kind":"virtioserial","LogFile":"/virtioserial","UsesStdio":false},{"kind":"virtioinput","inputType":"keyboard"},{"kind":"virtiogpu","usesGUI":false,"height":800,"width":600},{"kind":"virtionet","Nat":true,"MacAddress":"ABEiM0RV","Socket":null,"UnixSocketPath":""},{"kind":"virtiorng"},{"kind":"virtioblk","DevName":"virtio-blk","ImagePath":"/virtioblk","ReadOnly":false,"DeviceIdentifier":""},{"kind":"virtiosock","Port":1234,"SocketURL":"/virtiovsock","Listen":false},{"kind":"virtiofs","MountTag":"tag","SharedDir":"/virtiofs"},{"kind":"usbmassstorage","DevName":"usb-mass-storage","ImagePath":"/usbmassstorage","ReadOnly":false},{"kind":"rosetta","MountTag":"vz-rosetta","InstallRosetta":false}]}`,
},
}

Expand Down
58 changes: 56 additions & 2 deletions pkg/config/virtio.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,10 +61,20 @@ type VirtioBlk struct {
DeviceIdentifier string
}

type DirectorySharingConfig struct {
MountTag string
}

// VirtioFs configures directory sharing between the guest and the host.
type VirtioFs struct {
DirectorySharingConfig
SharedDir string
MountTag string
}

// RosettaShare configures rosetta support in the guest to run Intel binaries on Apple CPUs
type RosettaShare struct {
DirectorySharingConfig
InstallRosetta bool
}

// virtioRng configures a random number generator (RNG) device.
Expand Down Expand Up @@ -131,6 +141,8 @@ func deviceFromCmdLine(deviceOpts string) (VirtioDevice, error) {
}
var dev VirtioDevice
switch opts[0] {
case "rosetta":
dev = &RosettaShare{}
case "virtio-blk":
dev = virtioBlkNewEmpty()
case "virtio-fs":
Expand Down Expand Up @@ -545,8 +557,10 @@ func (dev *VirtioVsock) FromOptions(options []option) error {
// mounted in the VM using `mount -t virtiofs mountTag /some/dir`
func VirtioFsNew(sharedDir string, mountTag string) (VirtioDevice, error) {
return &VirtioFs{
DirectorySharingConfig: DirectorySharingConfig{
MountTag: mountTag,
},
SharedDir: sharedDir,
MountTag: mountTag,
}, nil
}

Expand Down Expand Up @@ -575,6 +589,46 @@ func (dev *VirtioFs) FromOptions(options []option) error {
return nil
}

// RosettaShare creates a new rosetta share for running x86_64 binaries on M1 machines.
// It will share a directory containing the linux rosetta binaries with the
// virtual machine. This directory can be mounted in the VM using `mount -t
// virtiofs mountTag /some/dir`
func RosettaShareNew(mountTag string) (VirtioDevice, error) {
return &RosettaShare{
DirectorySharingConfig: DirectorySharingConfig{
MountTag: mountTag,
},
}, nil
}

func (dev *RosettaShare) ToCmdLine() ([]string, error) {
if dev.MountTag == "" {
return nil, fmt.Errorf("rosetta shares require a mount tag to be specified")
}
builder := strings.Builder{}
builder.WriteString("rosetta")
fmt.Fprintf(&builder, ",mountTag=%s", dev.MountTag)
if dev.InstallRosetta {
builder.WriteString(",install")
}

return []string{"--device", builder.String()}, nil
}

func (dev *RosettaShare) FromOptions(options []option) error {
for _, option := range options {
switch option.key {
case "mountTag":
dev.MountTag = option.value
case "install":
dev.InstallRosetta = true
default:
return fmt.Errorf("Unknown option for rosetta share: %s", option.key)
}
}
return nil
}

type USBMassStorage struct {
StorageConfig
}
Expand Down
13 changes: 12 additions & 1 deletion pkg/config/virtio_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,22 @@ var virtioDevTests = map[string]virtioDevTest{
newDev: func() (VirtioDevice, error) { return VirtioFsNew("/foo/bar", "myTag") },
expectedDev: &VirtioFs{
SharedDir: "/foo/bar",
MountTag: "myTag",
DirectorySharingConfig: DirectorySharingConfig{
MountTag: "myTag",
},
},
expectedCmdLine: []string{"--device", "virtio-fs,sharedDir=/foo/bar,mountTag=myTag"},
alternateCmdLine: []string{"--device", "virtio-fs,mountTag=myTag,sharedDir=/foo/bar"},
},
"NewRosettaShare": {
newDev: func() (VirtioDevice, error) { return RosettaShareNew("myTag") },
expectedDev: &RosettaShare{
DirectorySharingConfig: DirectorySharingConfig{
MountTag: "myTag",
},
},
expectedCmdLine: []string{"--device", "rosetta,mountTag=myTag"},
},
"NewVirtioVsock": {
newDev: func() (VirtioDevice, error) { return VirtioVsockNew(1234, "/foo/bar.unix", false) },
expectedDev: &VirtioVsock{
Expand Down
9 changes: 9 additions & 0 deletions pkg/vf/rosetta_amd64.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package vf

import (
"fmt"
)

func (dev *RosettaShare) AddToVirtualMachineConfig(_ *vzVirtualMachineConfiguration) error {
return fmt.Errorf("rosetta is unsupported on non-arm64 platforms")
}
61 changes: 61 additions & 0 deletions pkg/vf/rosetta_arm64.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package vf

import (
"fmt"

"github.com/Code-Hex/vz/v3"
log "github.com/sirupsen/logrus"
)

func checkRosettaAvailability(install bool) error {
availability := vz.LinuxRosettaDirectoryShareAvailability()
switch availability {
case vz.LinuxRosettaAvailabilityNotSupported:
return fmt.Errorf("rosetta is not supported")
case vz.LinuxRosettaAvailabilityNotInstalled:
if !install {
return fmt.Errorf("rosetta is not installed")
}
log.Debugf("installing rosetta")
if err := vz.LinuxRosettaDirectoryShareInstallRosetta(); err != nil {
return fmt.Errorf("failed to install rosetta: %w", err)
}
log.Debugf("rosetta installed")
case vz.LinuxRosettaAvailabilityInstalled:
// nothing to do
}

return nil
}

func (dev *RosettaShare) toVz() (vz.DirectorySharingDeviceConfiguration, error) {
if dev.MountTag == "" {
return nil, fmt.Errorf("missing mandatory 'mountTage' option for rosetta share")
}
if err := checkRosettaAvailability(dev.InstallRosetta); err != nil {
return nil, err
}

rosettaShare, err := vz.NewLinuxRosettaDirectoryShare()
if err != nil {
return nil, fmt.Errorf("failed to create a new rosetta directory share: %w", err)
}
config, err := vz.NewVirtioFileSystemDeviceConfiguration(dev.MountTag)
if err != nil {
return nil, fmt.Errorf("failed to create a new virtio file system configuration for rosetta: %w", err)
}

config.SetDirectoryShare(rosettaShare)

return config, nil
}

func (dev *RosettaShare) AddToVirtualMachineConfig(vmConfig *vzVirtualMachineConfiguration) error {
fileSystemDeviceConfig, err := dev.toVz()
if err != nil {
return err
}
log.Infof("Adding virtio-fs device")
vmConfig.directorySharingDevicesConfiguration = append(vmConfig.directorySharingDevicesConfiguration, fileSystemDeviceConfig)
return nil
}
3 changes: 3 additions & 0 deletions pkg/vf/virtio.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"golang.org/x/sys/unix"
)

type RosettaShare config.RosettaShare
type VirtioBlk config.VirtioBlk
type VirtioFs config.VirtioFs
type VirtioRng config.VirtioRng
Expand Down Expand Up @@ -248,6 +249,8 @@ func AddToVirtualMachineConfig(dev config.VirtioDevice, vmConfig *vzVirtualMachi
return (*USBMassStorage)(d).AddToVirtualMachineConfig(vmConfig)
case *config.VirtioBlk:
return (*VirtioBlk)(d).AddToVirtualMachineConfig(vmConfig)
case *config.RosettaShare:
return (*RosettaShare)(d).AddToVirtualMachineConfig(vmConfig)
case *config.VirtioFs:
return (*VirtioFs)(d).AddToVirtualMachineConfig(vmConfig)
case *config.VirtioNet:
Expand Down

0 comments on commit 9a0f904

Please sign in to comment.