GCE: install containerized mounter on COS

The containerized mounter is a little tricky to install, with lots of
bind mounts.  This code path is only hit on GCE though.

nodeup/pkg/model/kubelet.go

@@ -18,8 +18,10 @@ package model
 
 import (
 	"fmt"
+	"path"
 	"path/filepath"
 
+	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
 	"k8s.io/kops/nodeup/pkg/distros"
 	"k8s.io/kops/pkg/apis/kops"
@@ -28,11 +30,12 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 	"k8s.io/kops/upup/pkg/fi/utils"
-
-	"github.com/golang/glog"
 )
 
-// KubeletBuilder install kubelet
+// containerizedMounterHome is the path where we install the containerized mounter (on ContainerOS)
+const containerizedMounterHome = "/home/kubernetes/containerized_mounter"
+
+// KubeletBuilder installs kubelet
 type KubeletBuilder struct {
 	*NodeupModelContext
 }
@@ -103,6 +106,10 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error {
 		return err
 	}
 
+	if err := b.addContainerizedMounter(c); err != nil {
+		return err
+	}
+
 	c.AddTask(b.buildSystemdService())
 
 	return nil
@@ -146,6 +153,11 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet
 		flags += " --network-plugin-dir=" + b.CNIBinDir()
 	}
 
+	if b.usesContainerizedMounter() {
+		// We don't want to expose this in the model while it is experimental, but it is needed on COS
+		flags += " --experimental-mounter-path=" + path.Join(containerizedMounterHome, "mounter")
+	}
+
 	sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n"
 	// Makes kubelet read /root/.docker/config.json properly
 	sysconfig = sysconfig + "HOME=\"/root" + "\"\n"
@@ -237,6 +249,106 @@ func (b *KubeletBuilder) addStaticUtils(c *fi.ModelBuilderContext) error {
 	return nil
 }
 
+// usesContainerizedMounter returns true if we use the containerized mounter
+func (b *KubeletBuilder) usesContainerizedMounter() bool {
+	switch b.Distribution {
+	case distros.DistributionContainerOS:
+		return true
+	default:
+		return false
+	}
+}
+
+// addContainerizedMounter downloads and installs the containerized mounter, that we need on ContainerOS
+func (b *KubeletBuilder) addContainerizedMounter(c *fi.ModelBuilderContext) error {
+	if !b.usesContainerizedMounter() {
+		return nil
+	}
+
+	// This is not a race because /etc is ephemeral on COS, and we start kubelet (also in /etc on COS)
+
+	// So what we do here is we download a tarred container image, expand it to containerizedMounterHome, then
+	// set up bind mounts so that the script is executable (most of containeros is noexec),
+	// and set up some bind mounts of proc and dev so that  mounting can take place inside that container
+	// - it isn't a full docker container.
+
+	{
+		// @TODO Extract to common function?
+		assetName := "gci-mounter"
+		assetPath := ""
+		asset, err := b.Assets.Find(assetName, assetPath)
+		if err != nil {
+			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
+		}
+		if asset == nil {
+			return fmt.Errorf("unable to locate asset %q", assetName)
+		}
+
+		t := &nodetasks.File{
+			Path:     path.Join(containerizedMounterHome, "mounter"),
+			Contents: asset,
+			Type:     nodetasks.FileType_File,
+			Mode:     s("0755"),
+		}
+		c.AddTask(t)
+	}
+
+	c.AddTask(&nodetasks.File{
+		Path: containerizedMounterHome,
+		Type: nodetasks.FileType_Directory,
+	})
+
+	// TODO: leverage assets for this tar file (but we want to avoid expansion of the archive)
+	c.AddTask(&nodetasks.Archive{
+		Name:      "containerized_mounter",
+		Source:    "https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar",
+		Hash:      "8003b798cf33c7f91320cd6ee5cec4fa22244571",
+		TargetDir: path.Join(containerizedMounterHome, "rootfs"),
+	})
+
+	c.AddTask(&nodetasks.File{
+		Path: path.Join(containerizedMounterHome, "rootfs/var/lib/kubelet"),
+		Type: nodetasks.FileType_Directory,
+	})
+
+	c.AddTask(&nodetasks.BindMount{
+		Source:     containerizedMounterHome,
+		Mountpoint: containerizedMounterHome,
+		Options:    []string{"exec"},
+	})
+
+	c.AddTask(&nodetasks.BindMount{
+		Source:     "/var/lib/kubelet/",
+		Mountpoint: path.Join(containerizedMounterHome, "rootfs/var/lib/kubelet"),
+		Options:    []string{"rshared"},
+		Recursive:  true,
+	})
+
+	c.AddTask(&nodetasks.BindMount{
+		Source:     "/proc",
+		Mountpoint: path.Join(containerizedMounterHome, "rootfs/proc"),
+		Options:    []string{"ro"},
+	})
+
+	c.AddTask(&nodetasks.BindMount{
+		Source:     "/dev",
+		Mountpoint: path.Join(containerizedMounterHome, "rootfs/dev"),
+		Options:    []string{"ro"},
+	})
+
+	// kube-up does a file cp, but we probably want to make changes visible (e.g. for gossip DNS)
+	c.AddTask(&nodetasks.BindMount{
+		Source:     "/etc/resolv.conf",
+		Mountpoint: path.Join(containerizedMounterHome, "rootfs/etc/resolv.conf"),
+		Options:    []string{"ro"},
+	})
+
+	//		cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/gci-mounter" "${CONTAINERIZED_MOUNTER_HOME}/mounter"
+	//		chmod a+x "${CONTAINERIZED_MOUNTER_HOME}/mounter"
+
+	return nil
+}
+
 const RoleLabelName15 = "kubernetes.io/role"
 const RoleLabelName16 = "kubernetes.io/role"
 const RoleMasterLabelValue15 = "master"

upup/pkg/fi/cloudup/apply_cluster.go

@@ -271,6 +271,17 @@ func (c *ApplyClusterCmd) Run() error {
 			}
 			c.Assets = append(c.Assets, hash.Hex()+"@"+utilsLocation)
 		}
+
+		if needsKubernetesManifests(cluster, c.InstanceGroups) {
+			defaultManifestsAsset := baseURL + "/kubernetes-manifests.tar.gz"
+			glog.V(2).Infof("Adding default kubernetes manifests asset: %s", defaultManifestsAsset)
+
+			hash, err := findHash(defaultManifestsAsset)
+			if err != nil {
+				return err
+			}
+			c.Assets = append(c.Assets, hash.Hex()+"@"+defaultManifestsAsset)
+		}
 	}
 
 	if c.NodeUpSource == "" {
@@ -1022,6 +1033,18 @@ func needsStaticUtils(c *kops.Cluster, instanceGroups []*kops.InstanceGroup) boo
 	return true
 }
 
+// needsKubernetesManifests checks if we need kubernetes manifests
+// This is only needed currently on ContainerOS i.e. GCE, but we don't have a nice way to detect it yet
+func needsKubernetesManifests(c *kops.Cluster, instanceGroups []*kops.InstanceGroup) bool {
+	// TODO: Do real detection of ContainerOS (but this has to work with AMI names, and maybe even forked AMIs)
+	switch kops.CloudProviderID(c.Spec.CloudProvider) {
+	case kops.CloudProviderGCE:
+		return true
+	default:
+		return false
+	}
+}
+
 func lifecyclePointer(v fi.Lifecycle) *fi.Lifecycle {
 	return &v
 }

upup/pkg/fi/nodeup/local/local_target.go

@@ -19,6 +19,7 @@ package local
 import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kops/upup/pkg/fi"
+	"os/exec"
 )
 
 type LocalTarget struct {
@@ -41,3 +42,9 @@ func (t *LocalTarget) HasTag(tag string) bool {
 	_, found := t.Tags[tag]
 	return found
 }
+
+// CombinedOutput is a helper function that executes a command, returning stdout & stderr combined
+func (t *LocalTarget) CombinedOutput(args []string) ([]byte, error) {
+	c := exec.Command(args[0], args[1:]...)
+	return c.CombinedOutput()
+}