From c84bebabf0a9e5ee8de62e132d91c959babaf34b Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 30 Sep 2017 02:05:03 -0400 Subject: [PATCH] GCE: install containerized mounter on COS The containerized mounter is a little tricky to install, with lots of bind mounts. This code path is only hit on GCE though. --- nodeup/pkg/model/kubelet.go | 118 +++++++- upup/pkg/fi/cloudup/apply_cluster.go | 23 ++ upup/pkg/fi/nodeup/local/local_target.go | 7 + upup/pkg/fi/nodeup/nodetasks/archive.go | 198 +++++++++++++ upup/pkg/fi/nodeup/nodetasks/archive_test.go | 62 +++++ upup/pkg/fi/nodeup/nodetasks/bindmount.go | 260 ++++++++++++++++++ .../pkg/fi/nodeup/nodetasks/bindmount_test.go | 253 +++++++++++++++++ upup/pkg/fi/nodeup/nodetasks/createsdir.go | 61 ++++ upup/pkg/fi/nodeup/nodetasks/file.go | 42 ++- upup/pkg/fi/nodeup/nodetasks/mount_disk.go | 23 +- 10 files changed, 1019 insertions(+), 28 deletions(-) create mode 100644 upup/pkg/fi/nodeup/nodetasks/archive.go create mode 100644 upup/pkg/fi/nodeup/nodetasks/archive_test.go create mode 100644 upup/pkg/fi/nodeup/nodetasks/bindmount.go create mode 100644 upup/pkg/fi/nodeup/nodetasks/bindmount_test.go create mode 100644 upup/pkg/fi/nodeup/nodetasks/createsdir.go diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index fbfe2e5b54f17..61fdcb107ad44 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -18,8 +18,10 @@ package model import ( "fmt" + "path" "path/filepath" + "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" @@ -28,11 +30,12 @@ import ( "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" "k8s.io/kops/upup/pkg/fi/utils" - - "github.com/golang/glog" ) -// KubeletBuilder install kubelet +// containerizedMounterHome is the path where we install the containerized mounter (on ContainerOS) +const containerizedMounterHome = "/home/kubernetes/containerized_mounter" + +// KubeletBuilder installs kubelet type KubeletBuilder struct { *NodeupModelContext } @@ -103,6 +106,10 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { return err } + if err := b.addContainerizedMounter(c); err != nil { + return err + } + c.AddTask(b.buildSystemdService()) return nil @@ -146,6 +153,11 @@ func (b *KubeletBuilder) buildSystemdEnvironmentFile(kubeletConfig *kops.Kubelet flags += " --network-plugin-dir=" + b.CNIBinDir() } + if b.usesContainerizedMounter() { + // We don't want to expose this in the model while it is experimental, but it is needed on COS + flags += " --experimental-mounter-path=" + path.Join(containerizedMounterHome, "mounter") + } + sysconfig := "DAEMON_ARGS=\"" + flags + "\"\n" // Makes kubelet read /root/.docker/config.json properly sysconfig = sysconfig + "HOME=\"/root" + "\"\n" @@ -237,6 +249,106 @@ func (b *KubeletBuilder) addStaticUtils(c *fi.ModelBuilderContext) error { return nil } +// usesContainerizedMounter returns true if we use the containerized mounter +func (b *KubeletBuilder) usesContainerizedMounter() bool { + switch b.Distribution { + case distros.DistributionContainerOS: + return true + default: + return false + } +} + +// addContainerizedMounter downloads and installs the containerized mounter, that we need on ContainerOS +func (b *KubeletBuilder) addContainerizedMounter(c *fi.ModelBuilderContext) error { + if !b.usesContainerizedMounter() { + return nil + } + + // This is not a race because /etc is ephemeral on COS, and we start kubelet (also in /etc on COS) + + // So what we do here is we download a tarred container image, expand it to containerizedMounterHome, then + // set up bind mounts so that the script is executable (most of containeros is noexec), + // and set up some bind mounts of proc and dev so that mounting can take place inside that container + // - it isn't a full docker container. + + { + // @TODO Extract to common function? + assetName := "gci-mounter" + assetPath := "" + asset, err := b.Assets.Find(assetName, assetPath) + if err != nil { + return fmt.Errorf("error trying to locate asset %q: %v", assetName, err) + } + if asset == nil { + return fmt.Errorf("unable to locate asset %q", assetName) + } + + t := &nodetasks.File{ + Path: path.Join(containerizedMounterHome, "mounter"), + Contents: asset, + Type: nodetasks.FileType_File, + Mode: s("0755"), + } + c.AddTask(t) + } + + c.AddTask(&nodetasks.File{ + Path: containerizedMounterHome, + Type: nodetasks.FileType_Directory, + }) + + // TODO: leverage assets for this tar file (but we want to avoid expansion of the archive) + c.AddTask(&nodetasks.Archive{ + Name: "containerized_mounter", + Source: "https://storage.googleapis.com/kubernetes-release/gci-mounter/mounter.tar", + Hash: "8003b798cf33c7f91320cd6ee5cec4fa22244571", + TargetDir: path.Join(containerizedMounterHome, "rootfs"), + }) + + c.AddTask(&nodetasks.File{ + Path: path.Join(containerizedMounterHome, "rootfs/var/lib/kubelet"), + Type: nodetasks.FileType_Directory, + }) + + c.AddTask(&nodetasks.BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }) + + c.AddTask(&nodetasks.BindMount{ + Source: "/var/lib/kubelet/", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/var/lib/kubelet"), + Options: []string{"rshared"}, + Recursive: true, + }) + + c.AddTask(&nodetasks.BindMount{ + Source: "/proc", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/proc"), + Options: []string{"ro"}, + }) + + c.AddTask(&nodetasks.BindMount{ + Source: "/dev", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/dev"), + Options: []string{"ro"}, + }) + + // kube-up does a file cp, but we probably want to make changes visible (e.g. for gossip DNS) + c.AddTask(&nodetasks.BindMount{ + Source: "/etc/resolv.conf", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/etc/resolv.conf"), + Options: []string{"ro"}, + }) + + // cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/gci-mounter" "${CONTAINERIZED_MOUNTER_HOME}/mounter" + // chmod a+x "${CONTAINERIZED_MOUNTER_HOME}/mounter" + + return nil +} + const RoleLabelName15 = "kubernetes.io/role" const RoleLabelName16 = "kubernetes.io/role" const RoleMasterLabelValue15 = "master" diff --git a/upup/pkg/fi/cloudup/apply_cluster.go b/upup/pkg/fi/cloudup/apply_cluster.go index 3eddfef899d5e..09d41864c5d30 100644 --- a/upup/pkg/fi/cloudup/apply_cluster.go +++ b/upup/pkg/fi/cloudup/apply_cluster.go @@ -271,6 +271,17 @@ func (c *ApplyClusterCmd) Run() error { } c.Assets = append(c.Assets, hash.Hex()+"@"+utilsLocation) } + + if needsKubernetesManifests(cluster, c.InstanceGroups) { + defaultManifestsAsset := baseURL + "/kubernetes-manifests.tar.gz" + glog.V(2).Infof("Adding default kubernetes manifests asset: %s", defaultManifestsAsset) + + hash, err := findHash(defaultManifestsAsset) + if err != nil { + return err + } + c.Assets = append(c.Assets, hash.Hex()+"@"+defaultManifestsAsset) + } } if c.NodeUpSource == "" { @@ -1022,6 +1033,18 @@ func needsStaticUtils(c *kops.Cluster, instanceGroups []*kops.InstanceGroup) boo return true } +// needsKubernetesManifests checks if we need kubernetes manifests +// This is only needed currently on ContainerOS i.e. GCE, but we don't have a nice way to detect it yet +func needsKubernetesManifests(c *kops.Cluster, instanceGroups []*kops.InstanceGroup) bool { + // TODO: Do real detection of ContainerOS (but this has to work with AMI names, and maybe even forked AMIs) + switch kops.CloudProviderID(c.Spec.CloudProvider) { + case kops.CloudProviderGCE: + return true + default: + return false + } +} + func lifecyclePointer(v fi.Lifecycle) *fi.Lifecycle { return &v } diff --git a/upup/pkg/fi/nodeup/local/local_target.go b/upup/pkg/fi/nodeup/local/local_target.go index aa0ee757c4812..d5d2f11088e48 100644 --- a/upup/pkg/fi/nodeup/local/local_target.go +++ b/upup/pkg/fi/nodeup/local/local_target.go @@ -19,6 +19,7 @@ package local import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kops/upup/pkg/fi" + "os/exec" ) type LocalTarget struct { @@ -41,3 +42,9 @@ func (t *LocalTarget) HasTag(tag string) bool { _, found := t.Tags[tag] return found } + +// CombinedOutput is a helper function that executes a command, returning stdout & stderr combined +func (t *LocalTarget) CombinedOutput(args []string) ([]byte, error) { + c := exec.Command(args[0], args[1:]...) + return c.CombinedOutput() +} diff --git a/upup/pkg/fi/nodeup/nodetasks/archive.go b/upup/pkg/fi/nodeup/nodetasks/archive.go new file mode 100644 index 0000000000000..f4389f998ac09 --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/archive.go @@ -0,0 +1,198 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetasks + +import ( + "encoding/json" + "fmt" + "github.com/golang/glog" + "io/ioutil" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/cloudinit" + "k8s.io/kops/upup/pkg/fi/nodeup/local" + "k8s.io/kops/util/pkg/hashing" + "os" + "os/exec" + "path" + "reflect" +) + +// Archive task downloads and extracts a tar file +type Archive struct { + Name string + + // Source is the location for the archive + Source string `json:"source,omitempty"` + // Hash is the source tar + Hash string `json:"hash,omitempty"` + + // TargetDir is the directory for extraction + TargetDir string `json:"target,omitempty"` +} + +const ( + localArchiveDir = "/var/cache/nodeup/archives/" + localArchiveStateDir = "/var/cache/nodeup/archives/state/" +) + +var _ fi.HasDependencies = &Archive{} + +// GetDependencies implements HasDependencies::GetDependencies +func (e *Archive) GetDependencies(tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + + // Requires parent directories to be created + for _, v := range findCreatesDirParents(e.TargetDir, tasks) { + deps = append(deps, v) + } + + return deps +} + +var _ fi.HasName = &Archive{} + +func (e *Archive) GetName() *string { + return &e.Name +} + +func (e *Archive) SetName(name string) { + e.Name = name +} + +// String returns a string representation, implementing the Stringer interface +func (e *Archive) String() string { + return fmt.Sprintf("Archive: %s %s->%s", e.Name, e.Source, e.TargetDir) +} + +var _ CreatesDir = &Archive{} + +// Dir implements CreatesDir::Dir +func (e *Archive) Dir() string { + return e.TargetDir +} + +func (e *Archive) Find(c *fi.Context) (*Archive, error) { + // We write a marker file to prevent re-execution + localStateFile := path.Join(localArchiveStateDir, e.Name) + stateBytes, err := ioutil.ReadFile(localStateFile) + if err != nil { + if os.IsNotExist(err) { + stateBytes = nil + } else { + glog.Warningf("error reading archive state %s: %v", localStateFile, err) + // We can just reinstall + return nil, nil + } + } + + if stateBytes == nil { + return nil, nil + } + + state := &Archive{} + if err := json.Unmarshal(stateBytes, state); err != nil { + glog.Warningf("error unmarshalling archive state %s: %v", localStateFile, err) + // We can just reinstall + return nil, nil + } + + if state.Hash == e.Hash && state.TargetDir == e.TargetDir { + return state, nil + } + return nil, nil +} + +func (e *Archive) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (_ *Archive) CheckChanges(a, e, changes *Archive) error { + return nil +} + +func (_ *Archive) RenderLocal(t *local.LocalTarget, a, e, changes *Archive) error { + if a == nil { + glog.Infof("Installing archive %q", e.Name) + + localFile := path.Join(localArchiveDir, e.Name) + if err := os.MkdirAll(localArchiveDir, 0755); err != nil { + return fmt.Errorf("error creating directories %q: %v", localArchiveDir, err) + } + + var hash *hashing.Hash + if e.Hash != "" { + parsed, err := hashing.FromString(e.Hash) + if err != nil { + return fmt.Errorf("error paring hash: %v", err) + } + hash = parsed + } + if _, err := fi.DownloadURL(e.Source, localFile, hash); err != nil { + return err + } + + targetDir := e.TargetDir + if err := os.MkdirAll(targetDir, 0755); err != nil { + return fmt.Errorf("error creating directories %q: %v", targetDir, err) + } + + args := []string{"tar", "xf", localFile, "-C", targetDir} + glog.Infof("running command %s", args) + cmd := exec.Command(args[0], args[1:]...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("error installing archive %q: %v: %s", e.Name, err, string(output)) + } + + // We write a marker file to prevent re-execution + localStateFile := path.Join(localArchiveStateDir, e.Name) + if err := os.MkdirAll(localArchiveStateDir, 0755); err != nil { + return fmt.Errorf("error creating directories %q: %v", localArchiveStateDir, err) + } + + state, err := json.MarshalIndent(e, "", " ") + if err != nil { + return fmt.Errorf("error marshalling archive state: %v", err) + } + + if err := ioutil.WriteFile(localStateFile, state, 0644); err != nil { + return fmt.Errorf("error writing archive state: %v", err) + } + } else { + if !reflect.DeepEqual(changes, &Archive{}) { + glog.Warningf("cannot apply archive changes for %q: %v", e.Name, changes) + } + } + + return nil +} + +func (_ *Archive) RenderCloudInit(t *cloudinit.CloudInitTarget, a, e, changes *Archive) error { + archiveName := e.Name + + localFile := path.Join(localArchiveDir, archiveName) + t.AddMkdirpCommand(localArchiveDir, 0755) + + targetDir := e.TargetDir + t.AddMkdirpCommand(targetDir, 0755) + + url := e.Source + t.AddDownloadCommand(cloudinit.Always, url, localFile) + + t.AddCommand(cloudinit.Always, "tar", "xf", localFile, "-C", targetDir) + + return nil +} diff --git a/upup/pkg/fi/nodeup/nodetasks/archive_test.go b/upup/pkg/fi/nodeup/nodetasks/archive_test.go new file mode 100644 index 0000000000000..148ace6b23b8d --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/archive_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetasks + +import ( + "k8s.io/kops/upup/pkg/fi" + "testing" +) + +func TestArchiveDependencies(t *testing.T) { + grid := []struct { + parent fi.Task + child fi.Task + }{ + { + parent: &MountDiskTask{ + Mountpoint: "/", + }, + child: &Archive{ + TargetDir: "/var/something", + }, + }, + { + parent: &Archive{ + TargetDir: "/var/something", + }, + child: &MountDiskTask{ + Mountpoint: "/var/something/subdir", + }, + }, + } + + for _, g := range grid { + allTasks := make(map[string]fi.Task) + allTasks["parent"] = g.parent + allTasks["child"] = g.child + + deps := g.parent.(fi.HasDependencies).GetDependencies(allTasks) + if len(deps) != 0 { + t.Errorf("found unexpected dependencies for parent: %v %v", g.parent, deps) + } + + childDeps := g.child.(fi.HasDependencies).GetDependencies(allTasks) + if len(childDeps) != 1 { + t.Errorf("found unexpected dependencies for child: %v %v", g.child, childDeps) + } + } +} diff --git a/upup/pkg/fi/nodeup/nodetasks/bindmount.go b/upup/pkg/fi/nodeup/nodetasks/bindmount.go new file mode 100644 index 0000000000000..2de7a5fe11bc7 --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/bindmount.go @@ -0,0 +1,260 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetasks + +import ( + "fmt" + "github.com/golang/glog" + "io/ioutil" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/cloudinit" + "k8s.io/kops/upup/pkg/fi/nodeup/local" + "strings" +) + +// BindMount performs bind mounts +type BindMount struct { + Source string `json:"source"` + Mountpoint string `json:"mountpoint"` + Options []string `json:"options,omitempty"` + Recursive bool `json:"recursive"` +} + +var _ fi.Task = &BindMount{} + +func (s *BindMount) String() string { + return fmt.Sprintf("BindMount: %s->%s", s.Source, s.Mountpoint) +} + +var _ CreatesDir = &BindMount{} + +// Dir implements CreatesDir::Dir +func (e *BindMount) Dir() string { + return e.Mountpoint +} + +var _ fi.HasName = &Archive{} + +func (e *BindMount) GetName() *string { + return fi.String("BindMount-" + e.Mountpoint) +} + +func (e *BindMount) SetName(name string) { + glog.Fatalf("SetName not supported for BindMount task") +} + +var _ fi.HasDependencies = &BindMount{} + +// GetDependencies implements HasDependencies::GetDependencies +func (e *BindMount) GetDependencies(tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + + // Requires parent directories to be created + for _, v := range findCreatesDirParents(e.Mountpoint, tasks) { + deps = append(deps, v) + } + for _, v := range findCreatesDirMatching(e.Mountpoint, tasks) { + if v != e && findTaskInSlice(deps, v) == -1 { + deps = append(deps, v) + } + } + + // Requires source to be created + for _, v := range findCreatesDirParents(e.Source, tasks) { + if findTaskInSlice(deps, v) == -1 { + deps = append(deps, v) + } + } + for _, v := range findCreatesDirMatching(e.Source, tasks) { + if v != e && findTaskInSlice(deps, v) == -1 { + deps = append(deps, v) + } + } + + return deps +} + +func findTaskInSlice(tasks []fi.Task, task fi.Task) int { + for i, t := range tasks { + if t == task { + return i + } + } + return -1 +} + +func (e *BindMount) Find(c *fi.Context) (*BindMount, error) { + mounts, err := ioutil.ReadFile("/proc/self/mountinfo") + if err != nil { + return nil, fmt.Errorf("error reading /proc/self/mountinfo: %v", err) + } + for _, line := range strings.Split(string(mounts), "\n") { + // See `man mount_namespaces` and `man proc` + // 534 458 8:1 /var/lib/kubelet /home/kubernetes/containerized_mounter/rootfs/var/lib/kubelet rw,nosuid,nodev,noexec,relatime shared:19 - ext4 /dev/sda1 rw,commit=30,data=ordered + line = strings.TrimSpace(line) + if line == "" { + continue + } + tokens := strings.Fields(line) + if len(tokens) < 8 { + glog.V(4).Infof("ignoring mountinfo line: %q", line) + } + + mountpoint := tokens[4] + if strings.TrimSuffix(mountpoint, "/") != strings.TrimSuffix(e.Mountpoint, "/") { + continue + } + + fstype := tokens[len(tokens)-3] + source := tokens[3] + switch fstype { + // Some special cases + case "devtmpfs": + source = "/dev" + case "proc": + source = "/proc" + } + if e.Source == "/etc/resolv.conf" { + // /etc/resolv.conf is a symlink on ContainerOS, and "magically" gets transformed to /systemd/resolve/resolv.conf + // Special case this very odd case! + if source == "/systemd/resolve/resolv.conf" || source == "/run/systemd/resolve/resolv.conf" { + source = e.Source // force match + } + } + if strings.TrimSuffix(source, "/") != strings.TrimSuffix(e.Source, "/") { + continue + } + + glog.V(8).Infof("candidate mount: %v", line) + + mountOptions := sets.NewString(strings.Split(tokens[5], ",")...) + // exec is inferred from a lack of noexec + if !mountOptions.Has("noexec") { + mountOptions.Insert("exec") + } + + // optional fields: zero or more fields of the form "tag[:value]" + for _, token := range tokens[6:] { + if token == "-" { + // the end of the optional fields is marked by a single hyphen. + break + } + if strings.HasPrefix(token, "shared:") { + mountOptions.Insert("rshared") + } + } + + if !mountOptions.HasAll(e.Options...) { + glog.V(2).Infof("options mismatch on mount %v", line) + continue + } + + glog.V(2).Infof("found matching mount %v", line) + a := &BindMount{ + Source: e.Source, + Mountpoint: e.Mountpoint, + Options: e.Options, + Recursive: e.Recursive, // TODO: Validate + } + return a, nil + } + + return nil, nil +} + +func (e *BindMount) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (s *BindMount) CheckChanges(a, e, changes *BindMount) error { + return nil +} + +// Executor allows execution of a command; it makes for testing of commands +type Executor interface { + CombinedOutput(args []string) ([]byte, error) +} + +func (_ *BindMount) RenderLocal(t *local.LocalTarget, a, e, changes *BindMount) error { + return e.execute(t) +} + +func (e *BindMount) execute(t Executor) error { + var simpleOptions []string + var makeOptions []string + var remountOptions []string + for _, option := range e.Options { + switch option { + case "ro": + simpleOptions = append(simpleOptions, "ro") + + case "rshared": + makeOptions = append(makeOptions, "--make-rshared") + + case "exec": + remountOptions = append(remountOptions, "exec") + + default: + return fmt.Errorf("unknown option: %q", option) + } + } + + { + args := []string{"mount"} + if e.Recursive { + args = append(args, "--rbind") + } else { + args = append(args, "--bind") + } + if len(simpleOptions) != 0 { + args = append(args, "-o", strings.Join(simpleOptions, ",")) + } + args = append(args, e.Source, e.Mountpoint) + + glog.Infof("running mount command %s", args) + if output, err := t.CombinedOutput(args); err != nil { + return fmt.Errorf("error doing mount %q: %v: %s", strings.Join(args, " "), err, string(output)) + } + } + + if len(remountOptions) != 0 { + args := []string{"mount", "-o", "remount," + strings.Join(remountOptions, ","), e.Mountpoint} + + glog.Infof("running mount command %s", args) + if output, err := t.CombinedOutput(args); err != nil { + return fmt.Errorf("error doing mount options %q: %v: %s", strings.Join(args, " "), err, string(output)) + } + } + + if len(makeOptions) != 0 { + args := []string{"mount"} + args = append(args, makeOptions...) + args = append(args, e.Mountpoint) + + glog.Infof("running mount command %s", args) + if output, err := t.CombinedOutput(args); err != nil { + return fmt.Errorf("error doing mount operation %q: %v: %s", strings.Join(args, " "), err, string(output)) + } + } + + return nil +} + +func (_ *BindMount) RenderCloudInit(t *cloudinit.CloudInitTarget, a, e, changes *BindMount) error { + return fmt.Errorf("BindMount::RenderCloudInit not implemented") +} diff --git a/upup/pkg/fi/nodeup/nodetasks/bindmount_test.go b/upup/pkg/fi/nodeup/nodetasks/bindmount_test.go new file mode 100644 index 0000000000000..8403fac18a665 --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/bindmount_test.go @@ -0,0 +1,253 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetasks + +import ( + "fmt" + "k8s.io/kops/upup/pkg/fi" + "path" + "reflect" + "strings" + "testing" +) + +func TestBindMountCommands(t *testing.T) { + containerizedMounterHome := "/containerized_mounter" + + grid := []struct { + mount *BindMount + executor *MockExecutor + }{ + { + mount: &BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }, + executor: &MockExecutor{ + Commands: []*MockCommand{ + {Args: []string{"mount", "--bind", "/containerized_mounter", "/containerized_mounter"}}, + {Args: []string{"mount", "-o", "remount,exec", "/containerized_mounter"}}, + }, + }, + }, + { + mount: &BindMount{ + Source: "/var/lib/kubelet/", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/var/lib/kubelet"), + Options: []string{"rshared"}, + Recursive: true, + }, + executor: &MockExecutor{ + Commands: []*MockCommand{ + {Args: []string{"mount", "--rbind", "/var/lib/kubelet/", "/containerized_mounter/rootfs/var/lib/kubelet"}}, + {Args: []string{"mount", "--make-rshared", "/containerized_mounter/rootfs/var/lib/kubelet"}}, + }, + }, + }, + { + mount: &BindMount{ + Source: "/proc", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/proc"), + Options: []string{"ro"}, + }, + executor: &MockExecutor{ + Commands: []*MockCommand{ + {Args: []string{"mount", "--bind", "-o", "ro", "/proc", "/containerized_mounter/rootfs/proc"}}, + }, + }, + }, + { + mount: &BindMount{ + Source: "/dev", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/dev"), + Options: []string{"ro"}, + }, + executor: &MockExecutor{ + Commands: []*MockCommand{ + {Args: []string{"mount", "--bind", "-o", "ro", "/dev", "/containerized_mounter/rootfs/dev"}}, + }, + }, + }, + { + mount: &BindMount{ + Source: "/etc/resolv.conf", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/etc/resolv.conf"), + Options: []string{"ro"}, + }, + executor: &MockExecutor{ + Commands: []*MockCommand{ + {Args: []string{"mount", "--bind", "-o", "ro", "/etc/resolv.conf", "/containerized_mounter/rootfs/etc/resolv.conf"}}, + }, + }, + }, + } + + for _, g := range grid { + err := g.mount.execute(g.executor) + if err != nil { + t.Errorf("unexpected error from %v: %v", g.mount, err) + } + if len(g.executor.Commands) != 0 { + t.Errorf("not all expected commands were called: %s", g.executor.Commands) + } + } +} + +func TestBindMountDependencies(t *testing.T) { + containerizedMounterHome := "/containerized_mounter" + + grid := []struct { + parent fi.Task + child fi.Task + }{ + { + parent: &MountDiskTask{ + Mountpoint: "/", + }, + child: &BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }, + }, + { + parent: &File{ + Path: containerizedMounterHome, + Type: FileType_Directory, + }, + child: &BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }, + }, + { + parent: &Archive{ + TargetDir: containerizedMounterHome, + }, + child: &BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }, + }, + { + parent: &BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }, + child: &BindMount{ + Source: "/var/lib/kubelet/", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/var/lib/kubelet"), + Options: []string{"rshared"}, + Recursive: true, + }, + }, + { + parent: &BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }, + child: &BindMount{ + Source: "/proc", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/proc"), + Options: []string{"ro"}, + }, + }, + { + parent: &BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }, + child: &BindMount{ + Source: "/dev", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/dev"), + Options: []string{"ro"}, + }, + }, + { + parent: &BindMount{ + Source: containerizedMounterHome, + Mountpoint: containerizedMounterHome, + Options: []string{"exec"}, + }, + child: &BindMount{ + Source: "/etc/resolv.conf", + Mountpoint: path.Join(containerizedMounterHome, "rootfs/etc/resolv.conf"), + Options: []string{"ro"}, + }, + }, + } + + for _, g := range grid { + allTasks := make(map[string]fi.Task) + allTasks["parent"] = g.parent + allTasks["child"] = g.child + + deps := g.parent.(fi.HasDependencies).GetDependencies(allTasks) + if len(deps) != 0 { + t.Errorf("found unexpected dependencies for parent: %v %v", g.parent, deps) + } + + childDeps := g.child.(fi.HasDependencies).GetDependencies(allTasks) + if len(childDeps) != 1 { + t.Errorf("found unexpected dependencies for child: %v %v", g.child, childDeps) + } + } +} + +// MockExecutor is a mock implementation of Executor +type MockExecutor struct { + Commands []*MockCommand +} + +type MockCommand struct { + Args []string + Result []byte + Error error +} + +func (c *MockCommand) String() string { + return strings.Join(c.Args, " ") +} + +var _ Executor = &MockExecutor{} + +func (m *MockExecutor) Expect(args []string) *MockCommand { + c := &MockCommand{ + Args: args, + } + m.Commands = append(m.Commands, c) + return c +} + +func (m *MockExecutor) CombinedOutput(args []string) ([]byte, error) { + key := strings.Join(args, " ") + if len(m.Commands) == 0 { + return nil, fmt.Errorf("unexpected command %q", key) + } + c := m.Commands[0] + if !reflect.DeepEqual(args, c.Args) { + return nil, fmt.Errorf("unexpected command %q", key) + } + m.Commands = m.Commands[1:] + return c.Result, c.Error +} diff --git a/upup/pkg/fi/nodeup/nodetasks/createsdir.go b/upup/pkg/fi/nodeup/nodetasks/createsdir.go new file mode 100644 index 0000000000000..57a043ee08ff3 --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/createsdir.go @@ -0,0 +1,61 @@ +package nodetasks + +import ( + "k8s.io/kops/upup/pkg/fi" + "strings" +) + +// CreatesDir is a marker interface for tasks that create directories, used for dependencies +type CreatesDir interface { + Dir() string +} + +var _ CreatesDir = &File{} + +// findCreatesDirParents finds the tasks which create parent directories for the given task +func findCreatesDirParents(p string, tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + for _, v := range tasks { + if createsDirectory, ok := v.(CreatesDir); ok { + dirPath := createsDirectory.Dir() + if dirPath != "" { + if !strings.HasSuffix(dirPath, "/") { + dirPath += "/" + } + + if p == dirPath { + continue + } + + if strings.HasPrefix(p, dirPath) { + deps = append(deps, v) + } + } + } + } + return deps +} + +// findCreatesDirMatching finds the tasks which create the specified directory (matching, non-recusive) +func findCreatesDirMatching(p string, tasks map[string]fi.Task) []fi.Task { + if !strings.HasSuffix(p, "/") { + p += "/" + } + + var deps []fi.Task + for _, v := range tasks { + if createsDirectory, ok := v.(CreatesDir); ok { + dirPath := createsDirectory.Dir() + if dirPath != "" { + if !strings.HasSuffix(dirPath, "/") { + dirPath += "/" + } + + if p == dirPath { + deps = append(deps, v) + } + } + } + } + return deps +} diff --git a/upup/pkg/fi/nodeup/nodetasks/file.go b/upup/pkg/fi/nodeup/nodetasks/file.go index b6f1b4a28237b..e27a02cf1968d 100644 --- a/upup/pkg/fi/nodeup/nodetasks/file.go +++ b/upup/pkg/fi/nodeup/nodetasks/file.go @@ -76,13 +76,14 @@ func NewFileTask(name string, src fi.Resource, destPath string, meta string) (*F var _ fi.HasDependencies = &File{} -func (f *File) GetDependencies(tasks map[string]fi.Task) []fi.Task { +// GetDependencies implements HasDependencies::GetDependencies +func (e *File) GetDependencies(tasks map[string]fi.Task) []fi.Task { var deps []fi.Task - if f.Owner != nil { - ownerTask := tasks["user/"+*f.Owner] + if e.Owner != nil { + ownerTask := tasks["user/"+*e.Owner] if ownerTask == nil { // The user might be a pre-existing user (e.g. admin) - glog.Warningf("Unable to find task %q", "user/"+*f.Owner) + glog.Warningf("Unable to find task %q", "user/"+*e.Owner) } else { deps = append(deps, ownerTask) } @@ -97,26 +98,9 @@ func (f *File) GetDependencies(tasks map[string]fi.Task) []fi.Task { } } - // Files depend on parent directories - for _, v := range tasks { - dir, ok := v.(*File) - if !ok { - continue - } - if dir.Type == FileType_Directory { - dirPath := dir.Path - if !strings.HasSuffix(dirPath, "/") { - dirPath += "/" - } - - if f.Path == dirPath { - continue - } - - if strings.HasPrefix(f.Path, dirPath) { - deps = append(deps, v) - } - } + // Requires parent directories to be created + for _, v := range findCreatesDirParents(e.Path, tasks) { + deps = append(deps, v) } return deps @@ -136,6 +120,16 @@ func (f *File) String() string { return fmt.Sprintf("File: %q", f.Path) } +var _ CreatesDir = &File{} + +// Dir implements CreatesDir::Dir +func (f *File) Dir() string { + if f.Type != FileType_Directory { + return "" + } + return f.Path +} + func findFile(p string) (*File, error) { stat, err := os.Lstat(p) if err != nil { diff --git a/upup/pkg/fi/nodeup/nodetasks/mount_disk.go b/upup/pkg/fi/nodeup/nodetasks/mount_disk.go index 481b2391238ca..10820ae4daa69 100644 --- a/upup/pkg/fi/nodeup/nodetasks/mount_disk.go +++ b/upup/pkg/fi/nodeup/nodetasks/mount_disk.go @@ -42,7 +42,28 @@ type MountDiskTask struct { var _ fi.Task = &MountDiskTask{} func (s *MountDiskTask) String() string { - return fmt.Sprintf("Disk: %s", s.Name) + return fmt.Sprintf("MountDisk: %s %s->%s", s.Name, s.Device, s.Mountpoint) +} + +var _ CreatesDir = &MountDiskTask{} + +// Dir implements CreatesDir::Dir +func (e *MountDiskTask) Dir() string { + return e.Mountpoint +} + +var _ fi.HasDependencies = &MountDiskTask{} + +// GetDependencies implements HasDependencies::GetDependencies +func (e *MountDiskTask) GetDependencies(tasks map[string]fi.Task) []fi.Task { + var deps []fi.Task + + // Requires parent directories to be created + for _, v := range findCreatesDirParents(e.Mountpoint, tasks) { + deps = append(deps, v) + } + + return deps } func NewMountDiskTask(name string, contents string, meta string) (fi.Task, error) {