From fe7f8f5fe48452c1a50bb39044c8e25933a86542 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Mon, 22 May 2017 13:19:25 -0700 Subject: [PATCH] Initial hyperkit driver implementation. --- Makefile | 9 + cmd/drivers/hyperkit/main.go | 10 + pkg/minikube/cluster/cluster.go | 5 + pkg/minikube/cluster/cluster_darwin.go | 16 ++ pkg/minikube/drivers/hyperkit/disk.go | 48 ++++ pkg/minikube/drivers/hyperkit/driver.go | 247 ++++++++++++++++++ pkg/minikube/drivers/hyperkit/iso.go | 70 +++++ pkg/minikube/drivers/hyperkit/network.go | 87 ++++++ .../drivers/plugin/register_driver.go | 3 +- 9 files changed, 494 insertions(+), 1 deletion(-) create mode 100644 cmd/drivers/hyperkit/main.go create mode 100644 pkg/minikube/drivers/hyperkit/disk.go create mode 100644 pkg/minikube/drivers/hyperkit/driver.go create mode 100644 pkg/minikube/drivers/hyperkit/iso.go create mode 100644 pkg/minikube/drivers/hyperkit/network.go diff --git a/Makefile b/Makefile index 46d60e151a7b..0b2378eb1349 100644 --- a/Makefile +++ b/Makefile @@ -193,6 +193,15 @@ out/minikube-installer.exe: out/minikube-windows-amd64.exe mv out/windows_tmp/minikube-installer.exe out/minikube-installer.exe rm -rf out/windows_tmp +out/docker-machine-driver-hyperkit: + go build -o $(BUILD_DIR)/hyperkit k8s.io/minikube/cmd/drivers/hyperkit + +.PHONY: install-hyperkit-driver +install-hyperkit-driver: out/docker-machine-driver-hyperkit + sudo cp out/hyperkit $(HOME)/bin/docker-machine-driver-hyperkit + sudo chown root:wheel $(HOME)/bin/docker-machine-driver-hyperkit + sudo chmod u+s $(HOME)/bin/docker-machine-driver-hyperkit + .PHONY: check-release check-release: go test -v ./deploy/minikube/release_sanity_test.go -tags=release diff --git a/cmd/drivers/hyperkit/main.go b/cmd/drivers/hyperkit/main.go new file mode 100644 index 000000000000..027ffc801a39 --- /dev/null +++ b/cmd/drivers/hyperkit/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/docker/machine/libmachine/drivers/plugin" + "k8s.io/minikube/pkg/minikube/drivers/hyperkit" +) + +func main() { + plugin.RegisterDriver(hyperkit.NewDriver("", "")) +} diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index 881dd47d6c20..c695f84cd9ef 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -34,6 +34,7 @@ import ( "github.com/docker/machine/libmachine/engine" "github.com/docker/machine/libmachine/host" "github.com/docker/machine/libmachine/mcnerror" + "github.com/docker/machine/libmachine/ssh" "github.com/docker/machine/libmachine/state" "github.com/golang/glog" "github.com/pkg/errors" @@ -56,6 +57,8 @@ const fileScheme = "file" //see: https://github.com/kubernetes/kubernetes/blob/master/pkg/util/logs/logs.go#L32-34 func init() { flag.Set("logtostderr", "false") + // Setting the default client to native gives much better performance. + ssh.SetDefaultClient(ssh.Native) } // StartHost starts a host VM. @@ -366,6 +369,8 @@ func createHost(api libmachine.API, config MachineConfig) (*host.Host, error) { driver = createHypervHost(config) case "none": driver = createNoneHost(config) + case "hyperkit": + driver = createHyperkitHost(config) default: glog.Exitf("Unsupported driver: %s\n", config.VMDriver) } diff --git a/pkg/minikube/cluster/cluster_darwin.go b/pkg/minikube/cluster/cluster_darwin.go index 85dfccd027a8..8cedadaf6c05 100644 --- a/pkg/minikube/cluster/cluster_darwin.go +++ b/pkg/minikube/cluster/cluster_darwin.go @@ -23,6 +23,7 @@ import ( "github.com/docker/machine/libmachine/drivers" cfg "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/drivers/hyperkit" ) func createVMwareFusionHost(config MachineConfig) drivers.Driver { @@ -57,6 +58,21 @@ type xhyveDriver struct { RawDisk bool } +func createHyperkitHost(config MachineConfig) *hyperkit.Driver { + return &hyperkit.Driver{ + BaseDriver: &drivers.BaseDriver{ + MachineName: cfg.GetMachineName(), + StorePath: constants.GetMinipath(), + SSHUser: "docker", + }, + Boot2DockerURL: config.Downloader.GetISOFileURI(config.MinikubeISO), + DiskSize: config.DiskSize, + Memory: config.Memory, + CPU: config.CPUs, + Cmdline: "loglevel=3 user=docker console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes base host=" + cfg.GetMachineName(), + } +} + func createXhyveHost(config MachineConfig) *xhyveDriver { useVirtio9p := !config.DisableDriverMounts return &xhyveDriver{ diff --git a/pkg/minikube/drivers/hyperkit/disk.go b/pkg/minikube/drivers/hyperkit/disk.go new file mode 100644 index 000000000000..d185391e5805 --- /dev/null +++ b/pkg/minikube/drivers/hyperkit/disk.go @@ -0,0 +1,48 @@ +package hyperkit + +import ( + "io/ioutil" + "os" + "path/filepath" + "syscall" + + "github.com/cloudflare/cfssl/log" + "github.com/docker/machine/libmachine/mcnutils" +) + +func createDiskImage(sshKeyPath, diskPath string, diskSize int) error { + tarBuf, err := mcnutils.MakeDiskImage(sshKeyPath) + if err != nil { + return err + } + + file, err := os.OpenFile(diskPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + file.Seek(0, os.SEEK_SET) + + if _, err := file.Write(tarBuf.Bytes()); err != nil { + return err + } + file.Close() + + if err := os.Truncate(diskPath, int64(diskSize*1048576)); err != nil { + return err + } + return nil +} + +func fixPermissions(path string) error { + os.Chown(path, syscall.Getuid(), syscall.Getegid()) + files, _ := ioutil.ReadDir(path) + for _, f := range files { + fp := filepath.Join(path, f.Name()) + log.Debugf(fp) + if err := os.Chown(fp, syscall.Getuid(), syscall.Getegid()); err != nil { + return err + } + } + return nil +} diff --git a/pkg/minikube/drivers/hyperkit/driver.go b/pkg/minikube/drivers/hyperkit/driver.go new file mode 100644 index 000000000000..a7889cbafee1 --- /dev/null +++ b/pkg/minikube/drivers/hyperkit/driver.go @@ -0,0 +1,247 @@ +package hyperkit + +import ( + "fmt" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/docker/machine/libmachine/drivers" + "github.com/docker/machine/libmachine/log" + "github.com/docker/machine/libmachine/mcnflag" + "github.com/docker/machine/libmachine/mcnutils" + "github.com/docker/machine/libmachine/ssh" + "github.com/docker/machine/libmachine/state" + hyperkit "github.com/moby/hyperkit/go" + "github.com/pborman/uuid" + vmnet "github.com/zchee/go-vmnet" + commonutil "k8s.io/minikube/pkg/util" +) + +const ( + isoFilename = "boot2docker.iso" +) + +type Driver struct { + *drivers.BaseDriver + Boot2DockerURL string + DiskSize int + Pid int + CPU int + Memory int + Cmdline string +} + +func NewDriver(hostName, storePath string) *Driver { + return &Driver{ + BaseDriver: &drivers.BaseDriver{ + SSHUser: "docker", + }, + } +} + +func (d *Driver) Create() error { + b2dutils := mcnutils.NewB2dUtils(d.StorePath) + + if err := ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil { + return err + } + + if err := b2dutils.CopyIsoToMachineDir(d.Boot2DockerURL, d.MachineName); err != nil { + return err + } + isoPath := d.ResolveStorePath(isoFilename) + if err := d.extractKernel(isoPath); err != nil { + return err + } + + return d.Start() +} + +// DriverName returns the name of the driver +func (d *Driver) DriverName() string { + return "hyperkit" +} + +// GetCreateFlags returns the mcnflag.Flag slice representing the flags +// that can be set, their descriptions and defaults. +func (d *Driver) GetCreateFlags() []mcnflag.Flag { + return nil +} + +// GetSSHHostname returns hostname for use with ssh +func (d *Driver) GetSSHHostname() (string, error) { + return d.IPAddress, nil +} + +// GetURL returns a Docker compatible host URL for connecting to this host +// e.g. tcp://1.2.3.4:2376 +func (d *Driver) GetURL() (string, error) { + ip, err := d.GetIP() + if err != nil { + return "", err + } + return fmt.Sprintf("tcp://%s:2376", ip), nil +} + +// GetState returns the state that the host is in (running, stopped, etc) +func (d *Driver) GetState() (state.State, error) { + if d.Pid == 0 { + return state.Stopped, nil + } + p, err := os.FindProcess(d.Pid) + if err != nil { + return state.Error, err + } + if p == nil { + return state.Stopped, nil + } + return state.Running, nil +} + +// Kill stops a host forcefully +func (d *Driver) Kill() error { + return d.sendSignal(syscall.SIGKILL) +} + +// PreCreateCheck allows for pre-create operations to make sure a driver is ready for creation +func (d *Driver) PreCreateCheck() error { + return nil +} + +// Remove a host +func (d *Driver) Remove() error { + s, err := d.GetState() + if err != nil || s == state.Error { + log.Infof("Error checking machine status: %s, assuming it has been removed already", err) + return nil + } + if s == state.Running { + if err := d.Stop(); err != nil { + return err + } + } + return nil +} + +// Restart a host. This may just call Stop(); Start() if the provider does not +// have any special restart behaviour. +func (d *Driver) Restart() error { + for _, f := range []func() error{d.Stop, d.Start} { + if err := f(); err != nil { + return err + } + } + return nil +} + +// SetConfigFromFlags configures the driver with the object that was returned +// by RegisterCreateFlags +func (d *Driver) SetConfigFromFlags(opts drivers.DriverOptions) error { + return nil +} + +// Start a host +func (d *Driver) Start() error { + + // TODO: handle the disk already existing. + // TODO: handle different disk types. + diskPath := filepath.Join(d.ResolveStorePath("."), d.MachineName+".rawdisk") + if err := createDiskImage(d.publicSSHKeyPath(), diskPath, d.DiskSize); err != nil { + return err + } + + if err := fixPermissions(d.ResolveStorePath(".")); err != nil { + return err + } + + h, err := hyperkit.New("", "", filepath.Join(d.StorePath, "machines", d.MachineName)) + if err != nil { + return err + } + + // TODO: handle the rest of our settings. + h.Kernel = d.ResolveStorePath("bzimage") + h.Initrd = d.ResolveStorePath("initrd") + h.VMNet = true + h.ISOImage = d.ResolveStorePath(isoFilename) + h.Console = hyperkit.ConsoleFile + h.CPUs = d.CPU + h.Memory = d.Memory + + // Set UUID + h.UUID = uuid.NewUUID().String() + log.Infof("Generated UUID %s", h.UUID) + mac, err := vmnet.GetMACAddressFromUUID(h.UUID) + if err != nil { + return err + } + + // Need to strip 0's + mac = trimMacAddress(mac) + log.Infof("Generated MAC %s", mac) + + h.Disks = []hyperkit.DiskConfig{ + { + Path: diskPath, + Size: d.DiskSize, + Driver: "virtio-blk", + }, + } + log.Infof("Starting with cmdline: %s", d.Cmdline) + if err := h.Start(d.Cmdline); err != nil { + return err + } + + d.Pid = h.Pid + + getIP := func() error { + var err error + d.IPAddress, err = GetIPAddressByMACAddress(mac) + if err != nil { + return &commonutil.RetriableError{Err: err} + } + return nil + } + + if err := commonutil.RetryAfter(30, getIP, 2*time.Second); err != nil { + return fmt.Errorf("IP address never found in dhcp leases file %v", err) + } + return nil +} + +// Stop a host gracefully +func (d *Driver) Stop() error { + return d.sendSignal(syscall.SIGTERM) +} + +func (d *Driver) extractKernel(isoPath string) error { + for _, f := range []struct { + pathInIso string + destPath string + }{ + {"/boot/bzimage", "bzimage"}, + {"/boot/initrd", "initrd"}, + {"/isolinux/isolinux.cfg", "isolinux.cfg"}, + } { + fullDestPath := d.ResolveStorePath(f.destPath) + if err := ExtractFile(isoPath, f.pathInIso, fullDestPath); err != nil { + return err + } + } + return nil +} + +func (d *Driver) publicSSHKeyPath() string { + return d.GetSSHKeyPath() + ".pub" +} + +func (d *Driver) sendSignal(s os.Signal) error { + proc, err := os.FindProcess(d.Pid) + if err != nil { + return err + } + + return proc.Signal(s) +} diff --git a/pkg/minikube/drivers/hyperkit/iso.go b/pkg/minikube/drivers/hyperkit/iso.go new file mode 100644 index 000000000000..05a95e296256 --- /dev/null +++ b/pkg/minikube/drivers/hyperkit/iso.go @@ -0,0 +1,70 @@ +package hyperkit + +import ( + "fmt" + "io" + "io/ioutil" + "os" + + "strings" + + "github.com/hooklift/iso9660" +) + +func ExtractFile(isoPath, srcPath, destPath string) error { + iso, err := os.Open(isoPath) + defer iso.Close() + if err != nil { + return err + } + + r, err := iso9660.NewReader(iso) + if err != nil { + return err + } + + f, err := findFile(r, srcPath) + if err != nil { + return err + } + + dst, err := os.Create(destPath) + defer dst.Close() + if err != nil { + return err + } + + _, err = io.Copy(dst, f.Sys().(io.Reader)) + return err +} + +func ReadFile(isoPath, srcPath string) (string, error) { + iso, err := os.Open(isoPath) + defer iso.Close() + if err != nil { + return "", err + } + + r, err := iso9660.NewReader(iso) + if err != nil { + return "", err + } + + f, err := findFile(r, srcPath) + if err != nil { + return "", err + } + + contents, err := ioutil.ReadAll(f.Sys().(io.Reader)) + return string(contents), err +} + +func findFile(r *iso9660.Reader, path string) (os.FileInfo, error) { + for f, err := r.Next(); err != io.EOF; f, err = r.Next() { + // Some files get an extra ',' at the end. + if strings.TrimSuffix(f.Name(), ".") == path { + return f, nil + } + } + return nil, fmt.Errorf("Unable to find file %s.", path) +} diff --git a/pkg/minikube/drivers/hyperkit/network.go b/pkg/minikube/drivers/hyperkit/network.go new file mode 100644 index 000000000000..6f8a4813fc69 --- /dev/null +++ b/pkg/minikube/drivers/hyperkit/network.go @@ -0,0 +1,87 @@ +package hyperkit + +import ( + "bufio" + "fmt" + "io" + "os" + "regexp" + "strings" +) + +const ( + DHCPLeasesFile = "/var/db/dhcpd_leases" +) + +type DHCPEntry struct { + Name string + IPAddress string + HWAddress string + ID string + Lease string +} + +func GetIPAddressByMACAddress(mac string) (string, error) { + file, err := os.Open(DHCPLeasesFile) + if err != nil { + return "", err + } + defer file.Close() + + dhcpEntries, err := parseDHCPdLeasesFile(file) + if err != nil { + return "", err + } + for _, dhcpEntry := range dhcpEntries { + if dhcpEntry.HWAddress == mac { + return dhcpEntry.IPAddress, nil + } + } + return "", fmt.Errorf("Could not find an IP address for %s", mac) +} + +func parseDHCPdLeasesFile(file io.Reader) ([]DHCPEntry, error) { + var ( + dhcpEntry *DHCPEntry + dhcpEntries []DHCPEntry + ) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "{" { + dhcpEntry = new(DHCPEntry) + continue + } else if line == "}" { + dhcpEntries = append(dhcpEntries, *dhcpEntry) + continue + } + + split := strings.SplitN(line, "=", 2) + key, val := split[0], split[1] + switch key { + case "name": + dhcpEntry.Name = val + case "ip_address": + dhcpEntry.IPAddress = val + case "hw_address": + // The mac addresses have a '1,' at the start. + dhcpEntry.HWAddress = val[2:] + case "identifier": + dhcpEntry.ID = val + case "lease": + dhcpEntry.Lease = val + default: + return dhcpEntries, fmt.Errorf("Unable to parse line: %s", line) + } + } + return dhcpEntries, scanner.Err() +} + +// trimMacAddress trimming "0" of the ten's digit +func trimMacAddress(rawUUID string) string { + re := regexp.MustCompile(`0([A-Fa-f0-9](:|$))`) + mac := re.ReplaceAllString(rawUUID, "$1") + + return mac +} diff --git a/vendor/github.com/docker/machine/libmachine/drivers/plugin/register_driver.go b/vendor/github.com/docker/machine/libmachine/drivers/plugin/register_driver.go index 27e0dbc33d4f..0b06fae84d4a 100644 --- a/vendor/github.com/docker/machine/libmachine/drivers/plugin/register_driver.go +++ b/vendor/github.com/docker/machine/libmachine/drivers/plugin/register_driver.go @@ -57,7 +57,8 @@ Please use this plugin through the main 'docker-machine' binary. continue case <-time.After(heartbeatTimeout): // TODO: Add heartbeat retry logic - os.Exit(1) + continue + // os.Exit(1) } } }