diff --git a/cmd/tetragon-vmtests-run/conf.go b/cmd/tetragon-vmtests-run/conf.go index e5b8ec05828..ca66788292b 100644 --- a/cmd/tetragon-vmtests-run/conf.go +++ b/cmd/tetragon-vmtests-run/conf.go @@ -12,24 +12,20 @@ import ( // NB: we should use lvh's RunConf to avoid duplicating code type RunConf struct { + runner.RunConf testImage string baseFname string - kernelFname string dontRebuildImage bool useTetragonTesterInit bool testerOut string qemuPrint bool justBoot bool justBuildImage bool - disableKVM bool - enableHVF bool btfFile string disableUnifiedCgroups bool - portForwards runner.PortForwards testerConf vmtests.Conf detailedResults bool keepAllLogs bool - rootDev string filesystems []QemuFS } diff --git a/cmd/tetragon-vmtests-run/main.go b/cmd/tetragon-vmtests-run/main.go index b835ebc1330..cdc49c5ff89 100644 --- a/cmd/tetragon-vmtests-run/main.go +++ b/cmd/tetragon-vmtests-run/main.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/cilium/little-vm-helper/pkg/arch" "github.com/cilium/little-vm-helper/pkg/runner" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -30,7 +31,7 @@ func main() { t0 := time.Now() var err error - rcnf.portForwards, err = runner.ParsePortForward(ports) + rcnf.ForwardedPorts, err = runner.ParsePortForward(ports) if err != nil { return fmt.Errorf("error parseing ports: %w", err) } @@ -84,10 +85,16 @@ func main() { return err } - qemuBin := "qemu-system-x86_64" - qemuArgs, err := buildQemuArgs(log, &rcnf) + rcnf.Image = rcnf.testImageFname() + + qemuBin, err := arch.QemuBinary() if err != nil { - return err + return fmt.Errorf("failed to get QEMU binary: %w", err) + } + + qemuArgs, err := buildQemuArgs(log, rcnf) + if err != nil { + return fmt.Errorf("failed to build qemu args: %w", err) } if rcnf.qemuPrint { @@ -104,6 +111,7 @@ func main() { fmt.Printf("%s\n", sb.String()) return nil } + fmt.Println(qemuArgs) // if we don't need to run tests, just exec() so that user will be able to // login to the VM. @@ -144,12 +152,11 @@ func main() { cmd.Flags().StringVar(&rcnf.baseFname, "base", "", "base image filename") cmd.MarkFlagRequired("base") cmd.Flags().StringVar(&rcnf.testImage, "name", "tetragon", "new vm (and basis for the image name). New vm image will be in the directory of the base image") - cmd.Flags().StringVar(&rcnf.kernelFname, "kernel", "", "kernel filename to boot with. (if empty no -kernel option will be passed to qemu)") + cmd.Flags().StringVar(&rcnf.KernelFname, "kernel", "", "kernel filename to boot with. (if empty no -kernel option will be passed to qemu)") cmd.Flags().BoolVar(&rcnf.dontRebuildImage, "dont-rebuild-image", false, "dont rebuild image") cmd.Flags().BoolVar(&rcnf.useTetragonTesterInit, "use-tetragon-init", false, "use tetragon-vmtests-init as init process in the VM") cmd.Flags().BoolVar(&rcnf.qemuPrint, "qemu-cmd-print", false, "Do not run the qemu command, just print it") - cmd.Flags().BoolVar(&rcnf.disableKVM, "qemu-disable-kvm", false, "Do not use KVM acceleration, even if /dev/kvm exists") - cmd.Flags().BoolVar(&rcnf.enableHVF, "qemu-enable-hvf", false, "Use hvf acceleration") + cmd.Flags().BoolVar(&rcnf.DisableKVM, "qemu-disable-kvm", false, "Do not use KVM acceleration, even if /dev/kvm exists") cmd.Flags().BoolVar(&rcnf.justBoot, "just-boot", false, "Do not actually run any tests. Just setup everything and start the VM. User will be able to login to the VM.") cmd.Flags().BoolVar(&rcnf.justBuildImage, "just-build-image", false, "Just build an image. Do not actually run any tests or boot the VM.") cmd.Flags().BoolVar(&rcnf.testerConf.NoPowerOff, "no-poweroff", false, "Do not poweroff the VM at the end of the run") @@ -161,9 +168,35 @@ func main() { cmd.Flags().StringArrayVarP(&ports, "port", "p", nil, "Forward a port (hostport[:vmport[:tcp|udp]])") cmd.Flags().StringVar(&rcnf.testerConf.KernelVer, "kernel-ver", "", "kenel version") cmd.Flags().BoolVar(&rcnf.detailedResults, "enable-detailed-results", false, "produce detailed results") - cmd.Flags().StringVar(&rcnf.rootDev, "root-dev", "vda", "type of root device (hda or vda)") + cmd.Flags().StringVar(&rcnf.RootDev, "root-dev", "vda", "type of root device (hda or vda)") if err := cmd.Execute(); err != nil { os.Exit(1) } } + +// buildQemuArgs calls lvh's runner.BuildQemuArgs and handles custom +// configuration from vmtests +func buildQemuArgs(log *logrus.Logger, rcnf RunConf) ([]string, error) { + if rcnf.KernelFname != "" { + if rcnf.disableUnifiedCgroups { + rcnf.KernelAppendArgs = append(rcnf.KernelAppendArgs, "systemd.unified_cgroup_hierarchy=0") + } + if rcnf.useTetragonTesterInit { + rcnf.KernelAppendArgs = append(rcnf.KernelAppendArgs, fmt.Sprintf("init=%s", TetragonTesterBin)) + } + } + rcnf.CPU = 2 + rcnf.Mem = "4G" + + qemuArgs, err := runner.BuildQemuArgs(log, &rcnf.RunConf) + if err != nil { + return nil, err + } + + for _, fs := range rcnf.filesystems { + qemuArgs = append(qemuArgs, fs.qemuArgs()...) + } + + return qemuArgs, nil +} diff --git a/cmd/tetragon-vmtests-run/qemu.go b/cmd/tetragon-vmtests-run/qemu.go deleted file mode 100644 index ebfb0a566d8..00000000000 --- a/cmd/tetragon-vmtests-run/qemu.go +++ /dev/null @@ -1,85 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// Copyright Authors of Tetragon - -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/sirupsen/logrus" -) - -func buildQemuArgs(log *logrus.Logger, rcnf *RunConf) ([]string, error) { - qemuArgs := []string{ - // no need for all the default devices - "-nodefaults", - // no need display (-nographics seems a bit slower) - "-display", "none", - // don't reboot, just exit - "-no-reboot", - // cpus, memory - "-smp", "2", "-m", "4G", - } - - if rcnf.enableHVF { - log.Info("HVF enabled") - qemuArgs = append(qemuArgs, "-accel", "hvf") - } else if !rcnf.disableKVM { - // quick-and-dirty kvm detection - if f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0755); err == nil { - qemuArgs = append(qemuArgs, "-enable-kvm", "-cpu", "kvm64") - f.Close() - } else { - log.Infof("KVM disabled (%v)", err) - } - } - - var kernelRoot string - switch rcnf.rootDev { - case "hda": - qemuArgs = append(qemuArgs, "-hda", rcnf.testImageFname()) - kernelRoot = "/dev/sda" - case "vda": - qemuArgs = append(qemuArgs, "-drive", fmt.Sprintf("file=%s,if=virtio,index=0,media=disk", rcnf.testImageFname())) - kernelRoot = "/dev/vda" - default: - return nil, fmt.Errorf("invalid root device: %s", rcnf.rootDev) - } - - if rcnf.kernelFname != "" { - appendArgs := []string{ - fmt.Sprintf("root=%s", kernelRoot), - "console=ttyS0", - "earlyprintk=ttyS0", - "panic=-1", - } - if rcnf.disableUnifiedCgroups { - appendArgs = append(appendArgs, "systemd.unified_cgroup_hierarchy=0") - } - if rcnf.useTetragonTesterInit { - appendArgs = append(appendArgs, fmt.Sprintf("init=%s", TetragonTesterBin)) - } - qemuArgs = append(qemuArgs, - "-kernel", rcnf.kernelFname, - "-append", strings.Join(appendArgs, " "), - ) - } - - // NB: not sure what the best option is here, this is from trial-and-error - qemuArgs = append(qemuArgs, - "-serial", "mon:stdio", - "-device", "virtio-serial-pci", - ) - - for _, fs := range rcnf.filesystems { - qemuArgs = append(qemuArgs, fs.qemuArgs()...) - } - - if len(rcnf.portForwards) > 0 { - qemuArgs = append(qemuArgs, rcnf.portForwards.QemuArgs()...) - } - - return qemuArgs, nil -} diff --git a/go.mod b/go.mod index 9e95cc6c832..7c0e16fb982 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/bombsimon/logrusr/v4 v4.1.0 github.com/cilium/cilium v1.15.3 github.com/cilium/ebpf v0.14.0 - github.com/cilium/little-vm-helper v0.0.17 + github.com/cilium/little-vm-helper v0.0.18-0.20240412153556-e8c483732639 github.com/cilium/lumberjack/v2 v2.3.0 github.com/cilium/tetragon/api v0.0.0-00010101000000-000000000000 github.com/cilium/tetragon/pkg/k8s v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index 170a6080570..723e4b24f38 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,10 @@ github.com/cilium/ebpf v0.14.0 h1:0PsxAjO6EjI1rcT+rkp6WcCnE0ZvfkXBYiMedJtrSUs= github.com/cilium/ebpf v0.14.0/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P1Hso= github.com/cilium/little-vm-helper v0.0.17 h1:uKS/wQSPeFqgZk6fFRhnreGvhuQCnWsZvqhkF/PS/OM= github.com/cilium/little-vm-helper v0.0.17/go.mod h1:2q3DGb/ptNd+jnenMpx0l++PX6r85FzvaTvZG31pGAQ= +github.com/cilium/little-vm-helper v0.0.18-0.20240412094937-f1c38b8ed8c0 h1:i84xd6IBX9I6gvVj61SvuCmg5T3cOxK3dt8uNNjfroE= +github.com/cilium/little-vm-helper v0.0.18-0.20240412094937-f1c38b8ed8c0/go.mod h1:2q3DGb/ptNd+jnenMpx0l++PX6r85FzvaTvZG31pGAQ= +github.com/cilium/little-vm-helper v0.0.18-0.20240412153556-e8c483732639 h1:m5weN2xOecWVYsEBFbY4He+h9IfS5yC5Bddt74+ecyM= +github.com/cilium/little-vm-helper v0.0.18-0.20240412153556-e8c483732639/go.mod h1:2q3DGb/ptNd+jnenMpx0l++PX6r85FzvaTvZG31pGAQ= github.com/cilium/lumberjack/v2 v2.3.0 h1:IhVJMvPpqDYmQzC0KDhAoy7KlaRsyOsZnT97Nsa3u0o= github.com/cilium/lumberjack/v2 v2.3.0/go.mod h1:yfbtPGmg4i//5oEqzaMxDqSWqgfZFmMoV70Mc2k6v0A= github.com/cilium/proxy v0.0.0-20231031145409-f19708f3d018 h1:R/QlThqx099hS6req1k2Q87fvLSRgCEicQGate9vxO4= diff --git a/vendor/github.com/cilium/little-vm-helper/pkg/runner/conf.go b/vendor/github.com/cilium/little-vm-helper/pkg/runner/conf.go new file mode 100644 index 00000000000..f444bab4859 --- /dev/null +++ b/vendor/github.com/cilium/little-vm-helper/pkg/runner/conf.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package runner + +import ( + "github.com/sirupsen/logrus" +) + +type RunConf struct { + // Image filename + Image string + // kernel filename to boot with. (if empty no -kernel option will be passed to qemu) + KernelFname string + // kernel append args to add when a kernel is passed to qemu + KernelAppendArgs []string + // Do not run the qemu command, just print it + QemuPrint bool + // Do not use KVM acceleration, even if /dev/kvm exists + DisableKVM bool + // Daemonize QEMU after initializing + Daemonize bool + // Log file for virtual console output + ConsoleLogFile string + + // Print qemu command before running it + Verbose bool + + // Disable the network connection to the VM + DisableNetwork bool + ForwardedPorts PortForwards + + Logger *logrus.Logger + + HostMount string + + SerialPort int + + CPU int + Mem string + // Kind of CPU to use (e.g. host or kvm64) + CPUKind string + + // RootDev is the type of device used for the root fs. Can be "hda" or "vda" + RootDev string + + QemuMonitorPort int +} + +func (rc *RunConf) testImageFname() string { + return rc.Image +} diff --git a/vendor/github.com/cilium/little-vm-helper/pkg/runner/qemu.go b/vendor/github.com/cilium/little-vm-helper/pkg/runner/qemu.go new file mode 100644 index 00000000000..90c4b0af775 --- /dev/null +++ b/vendor/github.com/cilium/little-vm-helper/pkg/runner/qemu.go @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package runner + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/cilium/little-vm-helper/pkg/arch" + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" +) + +func BuildQemuArgs(log *logrus.Logger, rcnf *RunConf) ([]string, error) { + qemuArgs := []string{ + // no need for all the default devices + "-nodefaults", + // no need display (-nographics seems a bit slower) + "-display", "none", + // don't reboot, just exit + "-no-reboot", + // cpus, memory + "-smp", fmt.Sprintf("%d", rcnf.CPU), "-m", rcnf.Mem, + } + + qemuArgs = arch.AppendArchSpecificQemuArgs(qemuArgs) + + // quick-and-dirty kvm detection + kvmEnabled := false + if !rcnf.DisableKVM { + if f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0755); err == nil { + qemuArgs = append(qemuArgs, "-enable-kvm") + f.Close() + kvmEnabled = true + } else { + log.Info("KVM disabled") + } + } + + qemuArgs = arch.AppendCPUKind(qemuArgs, kvmEnabled, rcnf.CPUKind) + + if rcnf.SerialPort != 0 { + qemuArgs = append(qemuArgs, + "-serial", + fmt.Sprintf("telnet:localhost:%d,server,nowait", rcnf.SerialPort)) + } + + if rcnf.ConsoleLogFile != "" { + qemuArgs = append(qemuArgs, + "-serial", + fmt.Sprintf("file:%s", rcnf.ConsoleLogFile)) + } + + var kernelRoot string + switch rcnf.RootDev { + case "hda": + qemuArgs = append(qemuArgs, "-hda", rcnf.testImageFname()) + kernelRoot = "/dev/sda" + case "vda": + qemuArgs = append(qemuArgs, "-drive", fmt.Sprintf("file=%s,if=virtio,index=0,media=disk", rcnf.testImageFname())) + kernelRoot = "/dev/vda" + default: + return nil, fmt.Errorf("invalid root device: %s", rcnf.RootDev) + } + + if rcnf.KernelFname != "" { + console, err := arch.Console() + if err != nil { + return nil, fmt.Errorf("failed retrieving console name: %w", err) + } + + appendArgs := []string{ + fmt.Sprintf("root=%s", kernelRoot), + fmt.Sprintf("console=%s", console), + "earlyprintk=ttyS0", + "panic=-1", + } + appendArgs = append(appendArgs, rcnf.KernelAppendArgs...) + qemuArgs = append(qemuArgs, + "-kernel", rcnf.KernelFname, + "-append", strings.Join(appendArgs, " "), + ) + } + + if !rcnf.DisableNetwork { + qemuArgs = append(qemuArgs, rcnf.ForwardedPorts.QemuArgs()...) + } + + if !rcnf.Daemonize { + qemuArgs = append(qemuArgs, + "-serial", "mon:stdio", + "-device", "virtio-serial-pci", + ) + } else { + qemuArgs = append(qemuArgs, "-daemonize") + } + + if rcnf.QemuMonitorPort != 0 { + arg := fmt.Sprintf("tcp:localhost:%d,server,nowait", rcnf.QemuMonitorPort) + qemuArgs = append(qemuArgs, "-monitor", arg) + } + + if len(rcnf.HostMount) > 0 { + qemuArgs = append(qemuArgs, + "-fsdev", fmt.Sprintf("local,id=host_id,path=%s,security_model=none", rcnf.HostMount), + "-device", "virtio-9p-pci,fsdev=host_id,mount_tag=host_mount", + ) + } + + return qemuArgs, nil +} + +func StartQemu(rcnf RunConf) error { + qemuBin, err := arch.QemuBinary() + if err != nil { + return fmt.Errorf("failed to retrieve Qemu binary: %w", err) + } + + qemuArgs, err := BuildQemuArgs(rcnf.Logger, &rcnf) + if err != nil { + return err + } + + if rcnf.QemuPrint || rcnf.Verbose { + var sb strings.Builder + sb.WriteString(qemuBin) + for _, arg := range qemuArgs { + sb.WriteString(" ") + if len(arg) > 0 && arg[0] == '-' { + sb.WriteString("\\\n\t") + } + sb.WriteString(arg) + } + + fmt.Printf("%s\n", sb.String()) + // We don't want to return early if running in verbose mode + if rcnf.QemuPrint { + return nil + } + } + + qemuPath, err := exec.LookPath(qemuBin) + if err != nil { + return err + } + + return unix.Exec(qemuPath, append([]string{qemuBin}, qemuArgs...), nil) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7099c018d30..c8c0603f029 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -227,7 +227,7 @@ github.com/cilium/ebpf/internal/unix github.com/cilium/ebpf/link github.com/cilium/ebpf/perf github.com/cilium/ebpf/rlimit -# github.com/cilium/little-vm-helper v0.0.17 +# github.com/cilium/little-vm-helper v0.0.18-0.20240412153556-e8c483732639 ## explicit; go 1.21.0 github.com/cilium/little-vm-helper/pkg/arch github.com/cilium/little-vm-helper/pkg/images