From d8480ac75b05741542fc1d562c2ead7f020a2277 Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Wed, 27 Jan 2016 07:22:25 +0100 Subject: [PATCH 1/4] Add a periodic cgroup fingerprinter --- client/fingerprint/cgroup.go | 82 ++++++++++++++++++++ client/fingerprint/cgroup_linux.go | 22 ++++++ client/fingerprint/cgroup_test.go | 100 +++++++++++++++++++++++++ client/fingerprint/cgroup_universal.go | 8 ++ client/fingerprint/fingerprint.go | 3 + 5 files changed, 215 insertions(+) create mode 100644 client/fingerprint/cgroup.go create mode 100644 client/fingerprint/cgroup_linux.go create mode 100644 client/fingerprint/cgroup_test.go create mode 100644 client/fingerprint/cgroup_universal.go diff --git a/client/fingerprint/cgroup.go b/client/fingerprint/cgroup.go new file mode 100644 index 00000000000..b1573f6a826 --- /dev/null +++ b/client/fingerprint/cgroup.go @@ -0,0 +1,82 @@ +package fingerprint + +import ( + "fmt" + "log" + "time" + + client "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad/structs" +) + +const ( + cgroupAvailable = "available" + cgroupUnavailable = "unavailable" +) + +type CGroupFingerprint struct { + logger *log.Logger + lastState string + mountPointDetector MountPointDetector +} + +// An interface to isolate calls to the cgroup library +// This facilitates testing where we can implement +// fake mount points to test varios code paths +type MountPointDetector interface { + MountPoint() (string, error) +} + +// Implements the interface detector which calls net directly +type DefaultMountPointDetector struct { +} + +// Call out to the default cgroup library +func (b *DefaultMountPointDetector) MountPoint() (string, error) { + return FindCgroupMountpointDir() +} + +func NewCGroupFingerprint(logger *log.Logger) Fingerprint { + f := &CGroupFingerprint{ + logger: logger, + lastState: cgroupUnavailable, + mountPointDetector: &DefaultMountPointDetector{}, + } + return f +} + +func (f *CGroupFingerprint) Fingerprint(cfg *client.Config, node *structs.Node) (bool, error) { + // Try to find a valid cgroup mount point + mount, err := f.mountPointDetector.MountPoint() + if err != nil { + return false, fmt.Errorf("Failed to discover cgroup mount point: %s", err) + } + + // Check if a cgroup mount point was found + if mount == "" { + // Clear any attributes from the previous fingerprint. + f.clearCGroupAttributes(node) + + if f.lastState == cgroupAvailable { + f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are unavailable") + } + f.lastState = cgroupUnavailable + return true, nil + } + + node.Attributes["cgroup.mountpoint"] = mount + + if f.lastState == cgroupUnavailable { + f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are available") + } + f.lastState = cgroupAvailable + return true, nil +} + +func (f *CGroupFingerprint) clearCGroupAttributes(n *structs.Node) { + delete(n.Attributes, "cgroup.mountpoint") +} + +func (f *CGroupFingerprint) Periodic() (bool, time.Duration) { + return true, 15 * time.Second +} diff --git a/client/fingerprint/cgroup_linux.go b/client/fingerprint/cgroup_linux.go new file mode 100644 index 00000000000..71357b6607b --- /dev/null +++ b/client/fingerprint/cgroup_linux.go @@ -0,0 +1,22 @@ +// +build linux + +package fingerprint + +import ( + "github.com/opencontainers/runc/libcontainer/cgroups" +) + +func FindCgroupMountpointDir() (string, error) { + mount, err := cgroups.FindCgroupMountpointDir() + if err != nil { + switch e := err.(type) { + case *cgroups.NotFoundError: + // It's okay if the mount point is not discovered + return "", nil + default: + // All other errors are passed back as is + return "", e + } + } + return mount, nil +} diff --git a/client/fingerprint/cgroup_test.go b/client/fingerprint/cgroup_test.go new file mode 100644 index 00000000000..d175ae46b9b --- /dev/null +++ b/client/fingerprint/cgroup_test.go @@ -0,0 +1,100 @@ +package fingerprint + +import ( + "fmt" + "testing" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/nomad/structs" +) + +// A fake mount point detector that returns an empty path +type MountPointDetectorNoMountPoint struct{} + +func (m *MountPointDetectorNoMountPoint) MountPoint() (string, error) { + return "", nil +} + +// A fake mount point detector that returns an error +type MountPointDetectorMountPointFail struct{} + +func (m *MountPointDetectorMountPointFail) MountPoint() (string, error) { + return "", fmt.Errorf("cgroup mountpoint discovery failed") +} + +// A fake mount point detector that returns a valid path +type MountPointDetectorValidMountPoint struct{} + +func (m *MountPointDetectorValidMountPoint) MountPoint() (string, error) { + return "/sys/fs/cgroup", nil +} + +// A fake mount point detector that returns an empty path +type MountPointDetectorEmptyMountPoint struct{} + +func (m *MountPointDetectorEmptyMountPoint) MountPoint() (string, error) { + return "", nil +} + +func TestCGroupFingerprint(t *testing.T) { + f := &CGroupFingerprint{ + logger: testLogger(), + lastState: cgroupUnavailable, + mountPointDetector: &MountPointDetectorMountPointFail{}, + } + + node := &structs.Node{ + Attributes: make(map[string]string), + } + + ok, err := f.Fingerprint(&config.Config{}, node) + if err == nil { + t.Fatalf("expected an error") + } + if ok { + t.Fatalf("should not apply") + } + if a, ok := node.Attributes["cgroup.mountpoint"]; ok { + t.Fatalf("unexpected attribute found, %s", a) + } + + f = &CGroupFingerprint{ + logger: testLogger(), + lastState: cgroupUnavailable, + mountPointDetector: &MountPointDetectorValidMountPoint{}, + } + + node = &structs.Node{ + Attributes: make(map[string]string), + } + + ok, err = f.Fingerprint(&config.Config{}, node) + if err != nil { + t.Fatalf("unexpected error, %s", err) + } + if !ok { + t.Fatalf("should apply") + } + assertNodeAttributeContains(t, node, "cgroup.mountpoint") + + f = &CGroupFingerprint{ + logger: testLogger(), + lastState: cgroupUnavailable, + mountPointDetector: &MountPointDetectorEmptyMountPoint{}, + } + + node = &structs.Node{ + Attributes: make(map[string]string), + } + + ok, err = f.Fingerprint(&config.Config{}, node) + if err != nil { + t.Fatalf("unexpected error, %s", err) + } + if !ok { + t.Fatalf("should apply") + } + if a, ok := node.Attributes["cgroup.mountpoint"]; ok { + t.Fatalf("unexpected attribute found, %s", a) + } +} diff --git a/client/fingerprint/cgroup_universal.go b/client/fingerprint/cgroup_universal.go new file mode 100644 index 00000000000..c884e91f34d --- /dev/null +++ b/client/fingerprint/cgroup_universal.go @@ -0,0 +1,8 @@ +// +build !linux + +package fingerprint + +// cgroups only exist on Linux +func FindCgroupMountpointDir() (string, error) { + return "", nil +} diff --git a/client/fingerprint/fingerprint.go b/client/fingerprint/fingerprint.go index a0139d485cb..3737d215e6d 100644 --- a/client/fingerprint/fingerprint.go +++ b/client/fingerprint/fingerprint.go @@ -16,6 +16,7 @@ const EmptyDuration = time.Duration(0) // fingerprints available, to provided an ordered iteration var BuiltinFingerprints = []string{ "arch", + "cgroup", "consul", "cpu", "env_aws", @@ -30,6 +31,7 @@ var BuiltinFingerprints = []string{ // which are available, corresponding to a key found in BuiltinFingerprints var builtinFingerprintMap = map[string]Factory{ "arch": NewArchFingerprint, + "cgroup": NewCGroupFingerprint, "consul": NewConsulFingerprint, "cpu": NewCPUFingerprint, "env_aws": NewEnvAWSFingerprint, @@ -41,6 +43,7 @@ var builtinFingerprintMap = map[string]Factory{ } // NewFingerprint is used to instantiate and return a new fingerprint + // given the name and a logger func NewFingerprint(name string, logger *log.Logger) (Fingerprint, error) { // Lookup the factory function From 117e035a982a6f4b7a26b29f8fd1a2bf878e04c6 Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Wed, 27 Jan 2016 07:23:02 +0100 Subject: [PATCH 2/4] Only enable exec driver when cgroups are available Make the exec driver fingerprinter periodic to disable the driver when cgroups become unavailable. --- client/driver/exec.go | 14 ++++++++------ client/driver/exec_test.go | 4 +++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/client/driver/exec.go b/client/driver/exec.go index e1fdb1646c3..4da62887991 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "path/filepath" - "runtime" "syscall" "time" @@ -13,7 +12,6 @@ import ( "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/executor" cstructs "github.com/hashicorp/nomad/client/driver/structs" - "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/client/getter" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" @@ -23,8 +21,8 @@ import ( // features. type ExecDriver struct { DriverContext - fingerprint.StaticFingerprinter } + type ExecDriverConfig struct { ArtifactSource string `mapstructure:"artifact_source"` Checksum string `mapstructure:"checksum"` @@ -47,9 +45,9 @@ func NewExecDriver(ctx *DriverContext) Driver { } func (d *ExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { - // Only enable if we are root on linux. - if runtime.GOOS != "linux" { - d.logger.Printf("[DEBUG] driver.exec: only available on linux, disabling") + // Only enable if cgroups are available and we are root + if _, ok := node.Attributes["cgroup.mountpoint"]; !ok { + d.logger.Printf("[DEBUG] driver.exec: cgroups unavailable, disabling") return false, nil } else if syscall.Geteuid() != 0 { d.logger.Printf("[DEBUG] driver.exec: must run as root user, disabling") @@ -60,6 +58,10 @@ func (d *ExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, return true, nil } +func (d *ExecDriver) Periodic() (bool, time.Duration) { + return true, 15 * time.Second +} + func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { var driverConfig ExecDriverConfig if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil { diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index 9c6cf864992..04d8dca89b1 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -22,7 +22,9 @@ func TestExecDriver_Fingerprint(t *testing.T) { driverCtx, _ := testDriverContexts(&structs.Task{Name: "foo"}) d := NewExecDriver(driverCtx) node := &structs.Node{ - Attributes: make(map[string]string), + Attributes: map[string]string{ + "cgroup.mountpoint": "/sys/fs/cgroup", + }, } apply, err := d.Fingerprint(&config.Config{}, node) if err != nil { From 5266f6d94780035b76f8bfcecb4b18b97bb0530a Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Wed, 27 Jan 2016 07:36:58 +0100 Subject: [PATCH 3/4] Fix comment --- client/fingerprint/cgroup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/fingerprint/cgroup.go b/client/fingerprint/cgroup.go index b1573f6a826..4137ae641d1 100644 --- a/client/fingerprint/cgroup.go +++ b/client/fingerprint/cgroup.go @@ -22,12 +22,12 @@ type CGroupFingerprint struct { // An interface to isolate calls to the cgroup library // This facilitates testing where we can implement -// fake mount points to test varios code paths +// fake mount points to test various code paths type MountPointDetector interface { MountPoint() (string, error) } -// Implements the interface detector which calls net directly +// Implements the interface detector which calls the cgroups library directly type DefaultMountPointDetector struct { } From 0289252ce7b0cbc36fea7762115857170163afe1 Mon Sep 17 00:00:00 2001 From: Ivo Verberk Date: Fri, 29 Jan 2016 14:34:29 +0100 Subject: [PATCH 4/4] Add comments and small improvements to cgroup fingerprinter --- client/driver/exec.go | 2 +- client/driver/exec_test.go | 2 +- client/fingerprint/cgroup.go | 14 ++++++++++---- client/fingerprint/cgroup_linux.go | 2 ++ client/fingerprint/cgroup_test.go | 6 +++--- client/fingerprint/cgroup_universal.go | 2 +- client/fingerprint/fingerprint.go | 3 +-- 7 files changed, 19 insertions(+), 12 deletions(-) diff --git a/client/driver/exec.go b/client/driver/exec.go index 4da62887991..efa846619a8 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -46,7 +46,7 @@ func NewExecDriver(ctx *DriverContext) Driver { func (d *ExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { // Only enable if cgroups are available and we are root - if _, ok := node.Attributes["cgroup.mountpoint"]; !ok { + if _, ok := node.Attributes["unique.cgroup.mountpoint"]; !ok { d.logger.Printf("[DEBUG] driver.exec: cgroups unavailable, disabling") return false, nil } else if syscall.Geteuid() != 0 { diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index 04d8dca89b1..5b077e9d699 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -23,7 +23,7 @@ func TestExecDriver_Fingerprint(t *testing.T) { d := NewExecDriver(driverCtx) node := &structs.Node{ Attributes: map[string]string{ - "cgroup.mountpoint": "/sys/fs/cgroup", + "unique.cgroup.mountpoint": "/sys/fs/cgroup", }, } apply, err := d.Fingerprint(&config.Config{}, node) diff --git a/client/fingerprint/cgroup.go b/client/fingerprint/cgroup.go index 4137ae641d1..3fb14a2f3b6 100644 --- a/client/fingerprint/cgroup.go +++ b/client/fingerprint/cgroup.go @@ -12,6 +12,7 @@ import ( const ( cgroupAvailable = "available" cgroupUnavailable = "unavailable" + interval = 15 ) type CGroupFingerprint struct { @@ -36,6 +37,7 @@ func (b *DefaultMountPointDetector) MountPoint() (string, error) { return FindCgroupMountpointDir() } +// NewCGroupFingerprint returns a new cgroup fingerprinter func NewCGroupFingerprint(logger *log.Logger) Fingerprint { f := &CGroupFingerprint{ logger: logger, @@ -45,10 +47,11 @@ func NewCGroupFingerprint(logger *log.Logger) Fingerprint { return f } +// Fingerprint tries to find a valid cgroup moint point func (f *CGroupFingerprint) Fingerprint(cfg *client.Config, node *structs.Node) (bool, error) { - // Try to find a valid cgroup mount point mount, err := f.mountPointDetector.MountPoint() if err != nil { + f.clearCGroupAttributes(node) return false, fmt.Errorf("Failed to discover cgroup mount point: %s", err) } @@ -64,7 +67,7 @@ func (f *CGroupFingerprint) Fingerprint(cfg *client.Config, node *structs.Node) return true, nil } - node.Attributes["cgroup.mountpoint"] = mount + node.Attributes["unique.cgroup.mountpoint"] = mount if f.lastState == cgroupUnavailable { f.logger.Printf("[INFO] fingerprint.cgroups: cgroups are available") @@ -73,10 +76,13 @@ func (f *CGroupFingerprint) Fingerprint(cfg *client.Config, node *structs.Node) return true, nil } +// clearCGroupAttributes clears any node attributes related to cgroups that might +// have been set in a previous fingerprint run. func (f *CGroupFingerprint) clearCGroupAttributes(n *structs.Node) { - delete(n.Attributes, "cgroup.mountpoint") + delete(n.Attributes, "unique.cgroup.mountpoint") } +// Periodic determines the interval at which the periodic fingerprinter will run. func (f *CGroupFingerprint) Periodic() (bool, time.Duration) { - return true, 15 * time.Second + return true, interval * time.Second } diff --git a/client/fingerprint/cgroup_linux.go b/client/fingerprint/cgroup_linux.go index 71357b6607b..25171c1f4c3 100644 --- a/client/fingerprint/cgroup_linux.go +++ b/client/fingerprint/cgroup_linux.go @@ -6,6 +6,8 @@ import ( "github.com/opencontainers/runc/libcontainer/cgroups" ) +// FindCgroupMountpointDir is used to find the cgroup mount point on a Linux +// system. func FindCgroupMountpointDir() (string, error) { mount, err := cgroups.FindCgroupMountpointDir() if err != nil { diff --git a/client/fingerprint/cgroup_test.go b/client/fingerprint/cgroup_test.go index d175ae46b9b..be5d251e080 100644 --- a/client/fingerprint/cgroup_test.go +++ b/client/fingerprint/cgroup_test.go @@ -54,7 +54,7 @@ func TestCGroupFingerprint(t *testing.T) { if ok { t.Fatalf("should not apply") } - if a, ok := node.Attributes["cgroup.mountpoint"]; ok { + if a, ok := node.Attributes["unique.cgroup.mountpoint"]; ok { t.Fatalf("unexpected attribute found, %s", a) } @@ -75,7 +75,7 @@ func TestCGroupFingerprint(t *testing.T) { if !ok { t.Fatalf("should apply") } - assertNodeAttributeContains(t, node, "cgroup.mountpoint") + assertNodeAttributeContains(t, node, "unique.cgroup.mountpoint") f = &CGroupFingerprint{ logger: testLogger(), @@ -94,7 +94,7 @@ func TestCGroupFingerprint(t *testing.T) { if !ok { t.Fatalf("should apply") } - if a, ok := node.Attributes["cgroup.mountpoint"]; ok { + if a, ok := node.Attributes["unique.cgroup.mountpoint"]; ok { t.Fatalf("unexpected attribute found, %s", a) } } diff --git a/client/fingerprint/cgroup_universal.go b/client/fingerprint/cgroup_universal.go index c884e91f34d..eeeade435d2 100644 --- a/client/fingerprint/cgroup_universal.go +++ b/client/fingerprint/cgroup_universal.go @@ -2,7 +2,7 @@ package fingerprint -// cgroups only exist on Linux +// FindCgroupMountpointDir returns an empty path on non-Linux systems func FindCgroupMountpointDir() (string, error) { return "", nil } diff --git a/client/fingerprint/fingerprint.go b/client/fingerprint/fingerprint.go index 3737d215e6d..c9195253ba3 100644 --- a/client/fingerprint/fingerprint.go +++ b/client/fingerprint/fingerprint.go @@ -12,7 +12,7 @@ import ( // EmptyDuration is to be used by fingerprinters that are not periodic. const EmptyDuration = time.Duration(0) -// BuiltinFingerprints is a slice containing the key names of all regestered +// BuiltinFingerprints is a slice containing the key names of all registered // fingerprints available, to provided an ordered iteration var BuiltinFingerprints = []string{ "arch", @@ -43,7 +43,6 @@ var builtinFingerprintMap = map[string]Factory{ } // NewFingerprint is used to instantiate and return a new fingerprint - // given the name and a logger func NewFingerprint(name string, logger *log.Logger) (Fingerprint, error) { // Lookup the factory function