From f23bce48218dfbc0fc922d4b93f1ebe2ed64f2c2 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 9 Apr 2020 21:39:05 +0000 Subject: [PATCH] Introduce kola qemuexec --devshell, make `cosa run` use it This finally unifies the advantages of `cosa run` and `kola spawn`. I kept getting annoyed by how serial console sizing is broken (e.g. trying to use `less` etc.). Using `ssh` via `kola spawn` addresses that, but it means you can't debug the initramfs. Now things work in an IMO pretty cool way; if you do e.g. `cosa run --kargs ignition.config.url=blah://` (or inject a bad Ignition config) to cause a failure in the initramfs, you'll see a nice error (building on https://github.com/coreos/ignition-dracut/pull/146 ) telling you to rerun with `cosa run --devshell-console`. Things are also wired up cleanly so that we support rebooting with the equivalent of `kola spawn --reconnect` (which we should probably remove now). You can exit via *either* quitting SSH cleanly or using `poweroff`, and the lifecycle of ssh and qemu is wired together. And finally, if we detect a cosa workdir we also bind it in by default. More to come here, such as auto-injecting debugging tools and containers. --- mantle/cmd/kola/devshell.go | 300 ++++++++++++++++++++++++++++++++++++ mantle/cmd/kola/qemuexec.go | 55 +++++-- mantle/platform/qemu.go | 13 ++ src/cmd-run | 18 ++- 4 files changed, 371 insertions(+), 15 deletions(-) create mode 100644 mantle/cmd/kola/devshell.go diff --git a/mantle/cmd/kola/devshell.go b/mantle/cmd/kola/devshell.go new file mode 100644 index 0000000000..9b2af94ea0 --- /dev/null +++ b/mantle/cmd/kola/devshell.go @@ -0,0 +1,300 @@ +// Copyright 2020 Red Hat, Inc. +// +// Run qemu as a development shell +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "fmt" + "io/ioutil" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/coreos/mantle/util" + "github.com/pkg/errors" + + v3 "github.com/coreos/ignition/v2/config/v3_0" + v3types "github.com/coreos/ignition/v2/config/v3_0/types" + + "github.com/coreos/mantle/kola" + "github.com/coreos/mantle/platform" +) + +const devshellHostname = "cosa-devsh" + +func devshellSSH(configPath, keyPath string, silent bool, args ...string) exec.Cmd { + sshArgs := append([]string{"-i", keyPath, "-F", configPath, devshellHostname}, args...) + sshCmd := exec.Command("ssh", sshArgs...) + if !silent { + sshCmd.Stdin = os.Stdin + sshCmd.Stdout = os.Stdout + sshCmd.Stderr = os.Stderr + } + sshCmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGTERM, + } + + return *sshCmd +} + +func readTrimmedLine(r *bufio.Reader) (string, error) { + l, err := r.ReadString('\n') + if err != nil { + return "", err + } + return strings.TrimSpace(l), nil +} + +func runDevShell(builder *platform.QemuBuilder, conf *v3types.Config) error { + tmpd, err := ioutil.TempDir("", "kola-devshell") + if err != nil { + return err + } + defer os.RemoveAll(tmpd) + + sshKeyPath := filepath.Join(tmpd, "ssh.key") + sshPubKeyPath := sshKeyPath + ".pub" + err = exec.Command("ssh-keygen", "-N", "", "-t", "ed25519", "-f", sshKeyPath).Run() + if err != nil { + return errors.Wrapf(err, "running ssh-keygen") + } + sshPubKeyBuf, err := ioutil.ReadFile(sshPubKeyPath) + if err != nil { + return errors.Wrapf(err, "reading pubkey") + } + sshPubKey := v3types.SSHAuthorizedKey(strings.TrimSpace(string(sshPubKeyBuf))) + + readinessSignalChan := "coreos.devshellready" + signalReadyUnit := fmt.Sprintf(`[Unit] +Requires=dev-virtio\\x2dports-%s.device +OnFailure=emergency.target +OnFailureJobMode=isolate +After=systemd-user-sessions.service +After=sshd.service +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/echo ready +StandardOutput=file:/dev/virtio-ports/%s +[Install] +RequiredBy=multi-user.target`, readinessSignalChan, readinessSignalChan) + signalPoweroffUnit := fmt.Sprintf(`[Unit] +Requires=dev-virtio\\x2dports-%s.device +OnFailure=emergency.target +OnFailureJobMode=isolate +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStop=/bin/echo poweroff +StandardOutput=file:/dev/virtio-ports/%s +[Install] +WantedBy=multi-user.target`, readinessSignalChan, readinessSignalChan) + + devshellConfig := v3types.Config{ + Ignition: v3types.Ignition{ + Version: "3.0.0", + }, + Passwd: v3types.Passwd{ + Users: []v3types.PasswdUser{ + { + Name: "core", + SSHAuthorizedKeys: []v3types.SSHAuthorizedKey{ + sshPubKey, + }, + }, + }, + }, + Systemd: v3types.Systemd{ + Units: []v3types.Unit{ + { + Name: "coreos-devshell-signal-ready.service", + Contents: &signalReadyUnit, + Enabled: util.BoolToPtr(true), + }, + { + Name: "coreos-devshell-signal-poweroff.service", + Contents: &signalPoweroffUnit, + Enabled: util.BoolToPtr(true), + }, + }, + }, + } + confm := v3.Merge(*conf, devshellConfig) + conf = &confm + + readyChan, err := builder.VirtioChannelRead(readinessSignalChan) + if err != nil { + return err + } + readyReader := bufio.NewReader(readyChan) + + if err := builder.SetConfig(*conf, kola.Options.IgnitionVersion == "v2"); err != nil { + return errors.Wrapf(err, "rendering config") + } + + builder.InheritConsole = devshellConsole + inst, err := builder.Exec() + if err != nil { + return err + } + defer inst.Destroy() + + if devshellConsole { + return inst.Wait() + } + + qemuWaitChan := make(chan error) + errchan := make(chan error) + readychan := make(chan struct{}) + go func() { + buf, err := inst.WaitIgnitionError() + if err != nil { + errchan <- err + } else { + // TODO parse buf and try to nicely render something + if buf != "" { + errchan <- platform.ErrInitramfsEmergency + } + } + }() + go func() { + qemuWaitChan <- inst.Wait() + }() + go func() { + readyMsg, err := readTrimmedLine(readyReader) + if err != nil { + errchan <- err + } + if readyMsg != "ready" { + errchan <- fmt.Errorf("Unexpected ready message: %s", readyMsg) + } + var s struct{} + readychan <- s + }() + + select { + case err := <-errchan: + if err == platform.ErrInitramfsEmergency { + return fmt.Errorf("instance failed in initramfs; try rerunning with --devshell-console") + } + return err + case err := <-qemuWaitChan: + return errors.Wrapf(err, "qemu exited before setup") + case _ = <-readychan: + fmt.Println("virtio: connected") + } + + var ip string + err = util.Retry(6, 5*time.Second, func() error { + var err error + ip, err = inst.SSHAddress() + if err != nil { + return err + } + return nil + }) + if err != nil { + return errors.Wrapf(err, "awaiting ssh address") + } + + sshConfigPath := filepath.Join(tmpd, "ssh-config") + sshConfig, err := os.OpenFile(sshConfigPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return errors.Wrapf(err, "creating ssh config") + } + defer sshConfig.Close() + sshBuf := bufio.NewWriter(sshConfig) + + _, err = fmt.Fprintf(sshBuf, "Host %s\n", devshellHostname) + if err != nil { + return err + } + host, port, err := net.SplitHostPort(ip) + if err != nil { + // Yeah this is hacky, surprising there's not a stdlib API for this + host = ip + port = "" + } + if port != "" { + if _, err := fmt.Fprintf(sshBuf, " Port %s\n", port); err != nil { + return err + } + } + if _, err := fmt.Fprintf(sshBuf, ` HostName %s + StrictHostKeyChecking no + UserKnownHostsFile /dev/null + User core + PasswordAuthentication no + KbdInteractiveAuthentication no + GSSAPIAuthentication no + IdentitiesOnly yes + ForwardAgent no + ForwardX11 no + `, host); err != nil { + return err + } + + if err := sshBuf.Flush(); err != nil { + return err + } + + err = util.Retry(10, 1*time.Second, func() error { + cmd := devshellSSH(sshConfigPath, sshKeyPath, true, "true") + return cmd.Run() + }) + if err != nil { + return err + } + + poweroffStarted := false + go func() { + msg, _ := readTrimmedLine(readyReader) + if msg == "poweroff" { + poweroffStarted = true + } + }() + + go func() { + for { + if poweroffStarted { + break + } + cmd := devshellSSH(sshConfigPath, sshKeyPath, false) + if err := cmd.Run(); err != nil { + fmt.Println("Disconnected, attempting to reconnect (Ctrl-C to exit)") + time.Sleep(1 * time.Second) + } + proc := os.Process{ + Pid: inst.Pid(), + } + poweroffStarted = true + proc.Signal(os.Interrupt) + break + } + }() + err = <-qemuWaitChan + if err == nil { + if !poweroffStarted { + fmt.Println("QEMU powered off unexpectedly") + } + } + return err +} diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index 72f957d9a8..81cf80f4ba 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -57,6 +57,9 @@ var ( directIgnition bool forceConfigInjection bool propagateInitramfsFailure bool + + devshell bool + devshellConsole bool ) func init() { @@ -69,6 +72,8 @@ func init() { cmdQemuExec.Flags().IntVarP(&memory, "memory", "m", 0, "Memory in MB") cmdQemuExec.Flags().BoolVar(&cpuCountHost, "auto-cpus", false, "Automatically set number of cpus to host count") cmdQemuExec.Flags().BoolVar(&directIgnition, "ignition-direct", false, "Do not parse Ignition, pass directly to instance") + cmdQemuExec.Flags().BoolVar(&devshell, "devshell", false, "Enable development shell") + cmdQemuExec.Flags().BoolVar(&devshellConsole, "devshell-console", false, "Connect directly to serial console in devshell mode") cmdQemuExec.Flags().StringVarP(&ignition, "ignition", "i", "", "Path to ignition config") cmdQemuExec.Flags().StringArrayVar(&bindro, "bind-ro", nil, "Mount readonly via 9pfs a host directory (use --bind-ro=/path/to/host,/var/mnt/guest") cmdQemuExec.Flags().StringArrayVar(&bindrw, "bind-rw", nil, "Same as above, but writable") @@ -102,6 +107,29 @@ func parseBindOpt(s string) (string, string, error) { func runQemuExec(cmd *cobra.Command, args []string) error { var err error var config *v3types.Config + + if devshellConsole { + devshell = true + } + if devshell { + if directIgnition { + return fmt.Errorf("Cannot use devshell with direct ignition") + } + ignitionFragments = append(ignitionFragments, "autologin") + cpuCountHost = true + usernet = true + if kola.Options.CosaWorkdir != "" { + // Conservatively bind readonly to avoid anything in the guest (stray tests, whatever) + // from destroying stuff + bindro = append(bindro, fmt.Sprintf("%s,/var/mnt/workdir", kola.Options.CosaWorkdir)) + // But provide the tempdir so it's easy to pass stuff back + bindrw = append(bindrw, fmt.Sprintf("%s,/var/mnt/workdir-tmp", kola.Options.CosaWorkdir+"/tmp")) + } + if hostname == "" { + hostname = devshellHostname + } + } + if ignition != "" && !directIgnition { buf, err := ioutil.ReadFile(ignition) if err != nil { @@ -148,16 +176,6 @@ func runQemuExec(cmd *cobra.Command, args []string) error { configv := v3.Merge(*config, conf.Mount9p(dest, false)) config = &configv } - if config != nil { - if directIgnition { - return fmt.Errorf("Cannot use fragments/mounts with direct ignition") - } - if err := builder.SetConfig(*config, kola.Options.IgnitionVersion == "v2"); err != nil { - return errors.Wrapf(err, "rendering config") - } - } else if directIgnition { - builder.ConfigFile = ignition - } builder.ForceConfigInjection = forceConfigInjection if len(knetargs) > 0 { builder.IgnitionNetworkKargs = knetargs @@ -196,9 +214,22 @@ func runQemuExec(cmd *cobra.Command, args []string) error { builder.EnableUsermodeNetworking(22) } builder.InheritConsole = true - builder.Append(args...) + if devshell { + return runDevShell(builder, config) + } + if config != nil { + if directIgnition { + return fmt.Errorf("Cannot use fragments/mounts with direct ignition") + } + if err := builder.SetConfig(*config, kola.Options.IgnitionVersion == "v2"); err != nil { + return errors.Wrapf(err, "rendering config") + } + } else if directIgnition { + builder.ConfigFile = ignition + } + inst, err := builder.Exec() if err != nil { return err @@ -206,7 +237,7 @@ func runQemuExec(cmd *cobra.Command, args []string) error { defer inst.Destroy() if propagateInitramfsFailure { - err = inst.WaitAll() + err := inst.WaitAll() if err != nil { return err } diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 421923e231..74d33fcf2f 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -853,6 +853,19 @@ func (builder *QemuBuilder) VirtioChannelRead(name string) (*os.File, error) { return r, nil } +// SerialPipe reads the serial console output into a pipe +func (builder *QemuBuilder) SerialPipe() (*os.File, error) { + r, w, err := os.Pipe() + if err != nil { + return nil, errors.Wrapf(err, "virtioChannelRead creating pipe") + } + id := "serialpipe" + builder.Append("-chardev", fmt.Sprintf("file,id=%s,path=%s,append=on", id, builder.AddFd(w))) + builder.Append("-serial", fmt.Sprintf("chardev:%s", id)) + + return r, nil +} + func (builder *QemuBuilder) Exec() (*QemuInstance, error) { builder.finalize() var err error diff --git a/src/cmd-run b/src/cmd-run index fb2c4b4647..a26b4325a2 100755 --- a/src/cmd-run +++ b/src/cmd-run @@ -1,5 +1,17 @@ #!/usr/bin/env bash set -euo pipefail -# This used to contain a lot of its own custom logic. All of the qemu code -# now lives inside mantle/kola. -exec kola qemuexec --hostname cosarun --add-ignition=autologin -U --auto-cpus "$@" +# This enables "devshell" mode. Important things to understand: +# +# The default connection is over ssh, but you can use --devshell-console +# to directly connect to the serial console to e.g. debug things in +# the initramfs. +# +# You can exit via either exiting the login session cleanly +# in SSH (ctrl-d/exit in bash), or via `poweroff`. +# +# If 9p is available and this is started from a coreos-assembler +# working directory, the build directory will be mounted readonly +# at /var/mnt/workdir, and the tmp/ directory will be read *write* +# at /var/mnt/workdir-tmp. This allows you to easily exchange +# data. +exec kola qemuexec --devshell "$@"