Add native arm64 support and cross-compilation for kernels

README.md

@@ -6,7 +6,7 @@ depend on the kernel, such as BPF. It is used in [cilium](https://github.com/cil
 meant, and should not be used for running production VMs. Fast booting and image building, as well
 as being storage efficient are the main goals.
 
-It uses [qemu](https://www.qemu.org/) and [libguestfs tools](https://libguestfs.org/).
+It uses [qemu](https://www.qemu.org/) and [libguestfs tools](https://libguestfs.org/). See [dependencies](#what-are-the-dependencies-of-lvh).
 
 Configurations for specific images used in the Cilium project can be found in:
 https://github.com/cilium/little-vm-helper-images.
@@ -88,6 +88,9 @@ $ go run ./cmd/lvh kernels --dir _data add bpf-next git://git.kernel.org/pub/scm
 $ go run ./cmd/lvh kernels --dir _data build bpf-next
 ```
 
+Please note, to cross-build for a different architecture, you can use the
+`--arch=arm64` or `--arch=amd64` flag.
+
 The configuration file keeps the url for a kernel, together with its configuration options:
 ```jsonc
 $ jq . < _data/kernel.json
@@ -123,7 +126,8 @@ bpf-next/
 git/
 ```
 
-Currently, kernels are built using the `bzImage` and `dir-pkg` targets (see [pkg/kernels/conf.go](pkg/kernels/conf.go)).
+Currently, kernels are built using the `bzImage` for x86\_64 or `Image.gz` for
+arm64, and `tar-pkg` targets (see [pkg/kernels/conf.go](pkg/kernels/conf.go)).
 
 ### Booting images
 
@@ -172,6 +176,18 @@ These tools also target production VMs with lifetime streching beyond a single
 use. As a result, they introduce overhead in booting time, provisioning time,
 and storage.
 
+### What are the dependencies of LVH?
+
+On debian distribution, here is a list of packages needed for LVH to work.
+
+| Action                         | Debian packages                                                                                                        |
+| --------                       | -------                                                                                                                |
+| Building images                | `qemu-kvm mmdebstrap debian-archive-keyring libguestfs-tools`                                                          |
+| Building the Linux kernel      | `libncurses-dev gawk flex bison openssl libssl-dev dkms libelf-dev libudev-dev libpci-dev libiberty-dev autoconf llvm` |
+| Cross-compile arm64 on x86\_64 | `gcc-aarch64-linux-gnu`                                                                                                |
+| Cross-compile x86\_64 on arm64 | `gcc-x86-64-linux-gnu`                                                                                                 |
+
+
 ### TODO
 
  - [ ] development workflow for MacOS X

cmd/lvh/kernels/build.go

@@ -12,14 +12,20 @@ import (
 )
 
 func buildCommand() *cobra.Command {
-	return &cobra.Command{
+	var arch string
+
+	cmd := &cobra.Command{
 		Use:   "build <kernel>",
 		Short: "build kernel",
 		Args:  cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			log := logrus.New()
 			kname := args[0]
-			return kernels.BuildKernel(context.Background(), log, dirName, kname, false /* TODO: add fetch flag */)
+			return kernels.BuildKernel(context.Background(), log, dirName, kname, false /* TODO: add fetch flag */, arch)
 		},
 	}
+
+	cmd.Flags().StringVar(&arch, "arch", "", "target architecture to build the kernel, e.g. 'amd64' or 'arm64' (default to native architecture)")
+
+	return cmd
 }

cmd/lvh/kernels/configure.go

@@ -12,7 +12,9 @@ import (
 )
 
 func configureCommand() *cobra.Command {
-	return &cobra.Command{
+	var arch string
+
+	cmd := &cobra.Command{
 		Use:   "configure <kernel>",
 		Short: "configure kernel",
 		Args:  cobra.ExactArgs(1),
@@ -24,12 +26,16 @@ func configureCommand() *cobra.Command {
 			}
 
 			kname := args[0]
-			if err := kd.ConfigureKernel(context.Background(), log, kname); err != nil {
+			if err := kd.ConfigureKernel(context.Background(), log, kname, arch); err != nil {
 				log.Fatal(err)
 			}
 
 		},
 	}
+
+	cmd.Flags().StringVar(&arch, "arch", "", "target architecture to configure the kernel, e.g. 'amd64' or 'arm64' (default to native architecture)")
+
+	return cmd
 }
 
 func rawConfigureCommand() *cobra.Command {

cmd/lvh/runner/qemu.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/cilium/little-vm-helper/pkg/arch"
 	"github.com/sirupsen/logrus"
 )
 
@@ -23,16 +24,22 @@ func BuildQemuArgs(log *logrus.Logger, rcnf *RunConf) ([]string, error) {
 		"-smp", fmt.Sprintf("%d", rcnf.CPU), "-m", rcnf.Mem,
 	}
 
+	qemuArgs = arch.AppendArchSpecificQemuArgs(qemuArgs)
+
 	// quick-and-dirty kvm detection
+	kvmEnabled := false
 	if !rcnf.DisableKVM {
 		if f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0755); err == nil {
-			qemuArgs = append(qemuArgs, "-enable-kvm", "-cpu", rcnf.CPUKind)
+			qemuArgs = append(qemuArgs, "-enable-kvm")
 			f.Close()
+			kvmEnabled = true
 		} else {
 			log.Info("KVM disabled")
 		}
 	}
 
+	qemuArgs = arch.AppendCPUKind(qemuArgs, kvmEnabled, rcnf.CPUKind)
+
 	if rcnf.SerialPort != 0 {
 		qemuArgs = append(qemuArgs,
 			"-serial",
@@ -58,9 +65,14 @@ func BuildQemuArgs(log *logrus.Logger, rcnf *RunConf) ([]string, error) {
 	}
 
 	if rcnf.KernelFname != "" {
+		console, err := arch.Console()
+		if err != nil {
+			return nil, fmt.Errorf("failed retrieving console name: %w", err)
+		}
+
 		appendArgs := []string{
 			fmt.Sprintf("root=%s", kernelRoot),
-			"console=ttyS0",
+			fmt.Sprintf("console=%s", console),
 			"earlyprintk=ttyS0",
 			"panic=-1",
 		}

cmd/lvh/runner/runner.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cilium/little-vm-helper/pkg/arch"
 	"github.com/cilium/little-vm-helper/pkg/images"
 	"github.com/cilium/little-vm-helper/pkg/runner"
 	"github.com/sirupsen/logrus"
@@ -92,17 +93,20 @@ func RunCommand() *cobra.Command {
 	cmd.Flags().IntVar(&rcnf.SerialPort, "serial-port", 0, "Port for serial console")
 	cmd.Flags().IntVar(&rcnf.CPU, "cpu", 2, "CPU count (-smp)")
 	cmd.Flags().StringVar(&rcnf.Mem, "mem", "4G", "RAM size (-m)")
-	cmd.Flags().StringVar(&rcnf.CPUKind, "cpu-kind", "kvm64", "CPU kind to use (-cpu), has no effect when KVM is disabled")
+	cmd.Flags().StringVar(&rcnf.CPUKind, "cpu-kind", "", "CPU kind to use (-cpu), has no effect when KVM is disabled (default 'kvm64' on amd64 and 'max' on arm64)")
 	cmd.Flags().IntVar(&rcnf.QemuMonitorPort, "qemu-monitor-port", 0, "Port for QEMU monitor")
 	cmd.Flags().StringVar(&rcnf.RootDev, "root-dev", "vda", "type of root device (hda or vda)")
 	cmd.Flags().BoolVarP(&rcnf.Verbose, "verbose", "v", false, "Print qemu command before running it")
 
 	return cmd
 }
 
-const qemuBin = "qemu-system-x86_64"
-
 func StartQemu(rcnf RunConf) error {
+	qemuBin, err := arch.QemuBinary()
+	if err != nil {
+		return fmt.Errorf("failed to retrieve Qemu binary: %w", err)
+	}
+
 	qemuArgs, err := BuildQemuArgs(rcnf.Logger, &rcnf)
 	if err != nil {
 		return err

pkg/arch/arch.go

@@ -0,0 +1,106 @@
+package arch
+
+import (
+	"fmt"
+	"runtime"
+)
+
+var ErrUnsupportedArch = fmt.Errorf("unsupported architecture: %s", runtime.GOARCH)
+
+// Target returns the Linux Makefile target to build the kernel, for historical
+// reasons, those are different between architectures.
+func Target(arch string) (string, error) {
+	if arch == "" {
+		arch = runtime.GOARCH
+	}
+	switch arch {
+	case "amd64":
+		return "bzImage", nil
+	case "arm64":
+		return "Image.gz", nil
+	default:
+		return "", fmt.Errorf("unsupported architecture for Makefile target: %s", arch)
+	}
+}
+
+func CrossCompiling(targetArch string) bool {
+	return targetArch != "" && targetArch != runtime.GOARCH
+}
+
+func CrossCompileMakeArgs(targetArch string) ([]string, error) {
+	if !CrossCompiling(targetArch) {
+		return nil, nil
+	}
+
+	switch targetArch {
+	case "arm64":
+		return []string{"ARCH=arm64", "CROSS_COMPILE=aarch64-linux-gnu-"}, nil
+	case "amd64":
+		return []string{"ARCH=x86_64", "CROSS_COMPILE=x86_64-linux-gnu-"}, nil
+	}
+	return nil, fmt.Errorf("unsupported architecture for cross-compilation: %s", targetArch)
+}
+
+func QemuBinary() (string, error) {
+	switch runtime.GOARCH {
+	case "amd64":
+		return "qemu-system-x86_64", nil
+	case "arm64":
+		return "qemu-system-aarch64", nil
+	default:
+		return "", ErrUnsupportedArch
+	}
+}
+
+// Console returns the name of the device for the first serial port.
+func Console() (string, error) {
+	switch runtime.GOARCH {
+	case "amd64":
+		return "ttyS0", nil
+	case "arm64":
+		return "ttyAMA0", nil
+	default:
+		return "", ErrUnsupportedArch
+	}
+}
+
+// AppendArchSpecificQemuArgs appends Qemu arguments to the input that are
+// specific to the architecture lvh is running on. For example on ARM64, Qemu
+// needs some precision on the -machine option to start.
+func AppendArchSpecificQemuArgs(qemuArgs []string) []string {
+	switch runtime.GOARCH {
+	case "arm64":
+		return append(qemuArgs, "-machine", "virt")
+	default:
+		return qemuArgs
+	}
+}
+
+// AppendCPUKind appends the -cpu type if needed, historically amd64 has used no
+// specific kind when running without KVM, and using kvm64 when running with
+// KVM. However, arm64 needs -cpu max in both cases to start properly.
+func AppendCPUKind(qemuArgs []string, kvmEnabled bool, cpuKind string) []string {
+	if cpuKind != "" {
+		return append(qemuArgs, "-cpu", cpuKind)
+	}
+	switch runtime.GOARCH {
+	case "amd64":
+		if kvmEnabled {
+			return append(qemuArgs, "-cpu", "kvm64")
+		}
+	case "arm64":
+		return append(qemuArgs, "-cpu", "max")
+	}
+	return qemuArgs
+}
+
+// Bootable returns the arch-dependent default value in case the pointer is nil,
+// so option is unconfigured. Typically arm64 should not be bootable by default
+// because we didn't take the time to find a bootloader that was arm64
+// compatible so far.
+func Bootable(bootable *bool) bool {
+	if bootable == nil {
+		return runtime.GOARCH == "amd64"
+	}
+	return *bootable
+}

pkg/images/conf.go

@@ -11,6 +11,9 @@ type ImgConf struct {
 	Parent string `json:"parent,omitempty"`
 	// ImageSize is the size of the image (defaults to images.DefaultImageSize)
 	ImageSize string `json:"image_size,omitempty"`
+	// Bootable indicates if the image should be bootable, i.e. contain a kernel
+	// and a bootloader.
+	Bootable *bool `json:"bootable,omitempty"`
 	// Packages is the list of packages contained in the image
 	Packages []string `json:"packages"`
 	// Actions is a list of additional actions for building the image.

pkg/images/step_create_image.go

@@ -5,13 +5,15 @@ package images
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"strings"
 
+	"github.com/cilium/little-vm-helper/pkg/arch"
 	"github.com/cilium/little-vm-helper/pkg/logcmd"
 	"github.com/cilium/little-vm-helper/pkg/step"
 	"github.com/sirupsen/logrus"
@@ -49,17 +51,11 @@ var (
 
 type CreateImage struct {
 	*StepConf
-	bootable bool
 }
 
 func NewCreateImage(cnf *StepConf) *CreateImage {
 	return &CreateImage{
 		StepConf: cnf,
-		// NB(kkourt): for now all the images we create are bootable because we can always
-		// boot them by directly specifing -kernel in qemu. Kept this, however, in case at
-		// some point we want to change it. Note, also, that because all images are
-		// bootable, it is sufficient to do create root bootable images.
-		bootable: true,
 	}
 }
 
@@ -74,11 +70,15 @@ append initrd=initrd.img root=%s rw console=ttyS0
 
 // makeRootImage creates a root (with respect to the image forest hierarch) image
 func (s *CreateImage) makeRootImage(ctx context.Context) error {
+	if s == nil || s.imgCnf == nil {
+		return errors.New("step configuration or image configuration is nil")
+	}
 	imgFname := filepath.Join(s.imagesDir, s.imgCnf.Name)
 	tarFname := path.Join(s.imagesDir, fmt.Sprintf("%s.tar", s.imgCnf.Name))
+	bootable := arch.Bootable(s.imgCnf.Bootable)
 	// build package list: add a kernel if building a bootable image
 	packages := make([]string, 0, len(s.imgCnf.Packages)+1)
-	if s.bootable {
+	if bootable {
 		packages = append(packages, "linux-image-amd64")
 	}
 	packages = append(packages, s.imgCnf.Packages...)
@@ -105,7 +105,7 @@ func (s *CreateImage) makeRootImage(ctx context.Context) error {
 	}
 
 	// example: guestfish -N foo.img=disk:8G -- mkfs ext4 /dev/vda : mount /dev/vda / : tar-in /tmp/foo.tar /
-	if s.bootable {
+	if bootable {
 		dirname, err := os.MkdirTemp("", "extlinux-")
 		if err != nil {
 			return err