From 84137b02c59aa1c3598fe26ed8d1efc4694dd44d Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Mon, 9 Jan 2017 11:21:51 -0800 Subject: [PATCH 1/3] Filter executor log messages --- client/config/config.go | 4 ++++ client/driver/docker.go | 2 +- client/driver/driver.go | 11 +++++++++-- client/driver/exec.go | 2 +- client/driver/executor/executor.go | 2 +- client/driver/java.go | 2 +- client/driver/plugins.go | 13 +++++++++++-- client/driver/qemu.go | 2 +- client/driver/raw_exec.go | 2 +- client/driver/rkt.go | 2 +- client/driver/utils.go | 27 +-------------------------- client/task_runner.go | 4 ++-- command/agent/agent.go | 1 + command/executor_plugin.go | 7 ++++--- command/syslog_plugin.go | 9 +++++---- 15 files changed, 44 insertions(+), 46 deletions(-) diff --git a/client/config/config.go b/client/config/config.go index d2806a8c769..d6bb591cedb 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -149,6 +149,9 @@ type Config struct { // TLSConfig holds various TLS related configurations TLSConfig *config.TLSConfig + + // LogLevel is the level of the logs to putout + LogLevel string } func (c *Config) Copy() *Config { @@ -172,6 +175,7 @@ func DefaultConfig() *Config { Region: "global", StatsCollectionInterval: 1 * time.Second, TLSConfig: &config.TLSConfig{}, + LogLevel: "DEBUG", } } diff --git a/client/driver/docker.go b/client/driver/docker.go index a740fc51f15..79db02673c1 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -394,7 +394,7 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile), + Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), } exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) diff --git a/client/driver/driver.go b/client/driver/driver.go index 893afa30042..97db2d1d4ec 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -152,11 +152,18 @@ type ExecContext struct { // Alloc ID AllocID string + + // LogLevel is the level of the logs to putout + LogLevel string } // NewExecContext is used to create a new execution context -func NewExecContext(td *allocdir.TaskDir, allocID string) *ExecContext { - return &ExecContext{TaskDir: td, AllocID: allocID} +func NewExecContext(td *allocdir.TaskDir, allocID string, logLevel string) *ExecContext { + return &ExecContext{ + TaskDir: td, + AllocID: allocID, + LogLevel: logLevel, + } } // GetTaskEnv converts the alloc dir, the node, task and alloc into a diff --git a/client/driver/exec.go b/client/driver/exec.go index ac96bd2600b..93e68cd1fa9 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -116,7 +116,7 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, } pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile), + Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), } exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) diff --git a/client/driver/executor/executor.go b/client/driver/executor/executor.go index 10a52cc05bb..3ff1f09d7ef 100644 --- a/client/driver/executor/executor.go +++ b/client/driver/executor/executor.go @@ -559,7 +559,7 @@ func (e *UniversalExecutor) pidStats() (map[string]*cstructs.ResourceUsage, erro for pid, np := range pids { p, err := process.NewProcess(int32(pid)) if err != nil { - e.logger.Printf("[DEBUG] executor: unable to create new process with pid: %v", pid) + e.logger.Printf("[TRACE] executor: unable to create new process with pid: %v", pid) continue } ms := &cstructs.MemoryStats{} diff --git a/client/driver/java.go b/client/driver/java.go index 6f2af363e46..4ce6b109339 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -198,7 +198,7 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile), + Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), } execIntf, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) diff --git a/client/driver/plugins.go b/client/driver/plugins.go index 4808d81e105..8818611b566 100644 --- a/client/driver/plugins.go +++ b/client/driver/plugins.go @@ -2,10 +2,13 @@ package driver import ( "io" + "io/ioutil" "log" "net" + "strings" "github.com/hashicorp/go-plugin" + "github.com/hashicorp/logutils" ) var HandshakeConfig = plugin.HandshakeConfig{ @@ -14,9 +17,15 @@ var HandshakeConfig = plugin.HandshakeConfig{ MagicCookieValue: "e4327c2e01eabfd75a8a67adb114fb34a757d57eee7728d857a8cec6e91a7255", } -func GetPluginMap(w io.Writer) map[string]plugin.Plugin { +func GetPluginMap(w io.Writer, logLevel string) map[string]plugin.Plugin { e := new(ExecutorPlugin) - e.logger = log.New(w, "", log.LstdFlags) + filter := &logutils.LevelFilter{ + Levels: []logutils.LogLevel{"TRACE", "DEBUG", "INFO", "WARN", "ERR"}, + MinLevel: logutils.LogLevel(strings.ToUpper(logLevel)), + Writer: ioutil.Discard, + } + + e.logger = log.New(filter, "", log.LstdFlags) s := new(SyslogCollectorPlugin) s.logger = log.New(w, "", log.LstdFlags) diff --git a/client/driver/qemu.go b/client/driver/qemu.go index 9ad850c3259..f2fc0eee40b 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -240,7 +240,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile), + Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), } exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index 972afe9e064..942a9d57e6c 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -130,7 +130,7 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl } pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile), + Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), } exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) diff --git a/client/driver/rkt.go b/client/driver/rkt.go index bf4b7d204b9..3fa1be50b51 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -401,7 +401,7 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e pluginLogFile := filepath.Join(ctx.TaskDir.Dir, fmt.Sprintf("%s-executor.out", task.Name)) pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile), + Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), } execIntf, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) diff --git a/client/driver/utils.go b/client/driver/utils.go index 0e6fc2883d0..9fc465bb391 100644 --- a/client/driver/utils.go +++ b/client/driver/utils.go @@ -13,7 +13,6 @@ import ( "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/executor" - "github.com/hashicorp/nomad/client/driver/logging" cstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/nomad/structs" ) @@ -23,7 +22,7 @@ import ( func createExecutor(config *plugin.ClientConfig, w io.Writer, clientConfig *config.Config) (executor.Executor, *plugin.Client, error) { config.HandshakeConfig = HandshakeConfig - config.Plugins = GetPluginMap(w) + config.Plugins = GetPluginMap(w, clientConfig.LogLevel) config.MaxPort = clientConfig.ClientMaxPort config.MinPort = clientConfig.ClientMinPort @@ -47,30 +46,6 @@ func createExecutor(config *plugin.ClientConfig, w io.Writer, return executorPlugin, executorClient, nil } -func createLogCollector(config *plugin.ClientConfig, w io.Writer, - clientConfig *config.Config) (logging.LogCollector, *plugin.Client, error) { - config.HandshakeConfig = HandshakeConfig - config.Plugins = GetPluginMap(w) - config.MaxPort = clientConfig.ClientMaxPort - config.MinPort = clientConfig.ClientMinPort - if config.Cmd != nil { - isolateCommand(config.Cmd) - } - - syslogClient := plugin.NewClient(config) - rpcCLient, err := syslogClient.Client() - if err != nil { - return nil, nil, fmt.Errorf("error creating rpc client for syslog plugin: %v", err) - } - - raw, err := rpcCLient.Dispense("syslogcollector") - if err != nil { - return nil, nil, fmt.Errorf("unable to dispense the syslog plugin: %v", err) - } - logCollector := raw.(logging.LogCollector) - return logCollector, syslogClient, nil -} - func consulContext(clientConfig *config.Config, containerID string) *executor.ConsulContext { return &executor.ConsulContext{ ConsulConfig: clientConfig.ConsulConfig, diff --git a/client/task_runner.go b/client/task_runner.go index f8331f80384..48f9c6207d6 100644 --- a/client/task_runner.go +++ b/client/task_runner.go @@ -267,7 +267,7 @@ func (r *TaskRunner) RestoreState() error { return err } - ctx := driver.NewExecContext(r.taskDir, r.alloc.ID) + ctx := driver.NewExecContext(r.taskDir, r.alloc.ID, r.config.LogLevel) handle, err := d.Open(ctx, snap.HandleID) // In the case it fails, we relaunch the task in the Run() method. @@ -1094,7 +1094,7 @@ func (r *TaskRunner) startTask() error { } // Run prestart - ctx := driver.NewExecContext(r.taskDir, r.alloc.ID) + ctx := driver.NewExecContext(r.taskDir, r.alloc.ID, r.config.LogLevel) if err := drv.Prestart(ctx, r.task); err != nil { wrapped := fmt.Errorf("failed to initialize task %q for alloc %q: %v", r.task.Name, r.alloc.ID, err) diff --git a/command/agent/agent.go b/command/agent/agent.go index e684365f5e2..6d482e1f1ec 100644 --- a/command/agent/agent.go +++ b/command/agent/agent.go @@ -200,6 +200,7 @@ func (a *Agent) clientConfig() (*clientconfig.Config, error) { conf.RPCHandler = a.server } conf.LogOutput = a.logOutput + conf.LogLevel = a.config.LogLevel conf.DevMode = a.config.DevMode if a.config.Region != "" { conf.Region = a.config.Region diff --git a/command/executor_plugin.go b/command/executor_plugin.go index 666cab96150..24a32853419 100644 --- a/command/executor_plugin.go +++ b/command/executor_plugin.go @@ -25,11 +25,12 @@ func (e *ExecutorPluginCommand) Synopsis() string { } func (e *ExecutorPluginCommand) Run(args []string) int { - if len(args) == 0 { - e.Ui.Error("log output file isn't provided") + if len(args) != 2 { + e.Ui.Error("log output file and log level are not provided") return 1 } logFileName := args[0] + logLevel := args[1] stdo, err := os.OpenFile(logFileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) if err != nil { e.Ui.Error(err.Error()) @@ -37,7 +38,7 @@ func (e *ExecutorPluginCommand) Run(args []string) int { } plugin.Serve(&plugin.ServeConfig{ HandshakeConfig: driver.HandshakeConfig, - Plugins: driver.GetPluginMap(stdo), + Plugins: driver.GetPluginMap(stdo, logLevel), }) return 0 } diff --git a/command/syslog_plugin.go b/command/syslog_plugin.go index 3afbc39a0f1..6b753ac15ce 100644 --- a/command/syslog_plugin.go +++ b/command/syslog_plugin.go @@ -21,15 +21,16 @@ func (e *SyslogPluginCommand) Help() string { } func (s *SyslogPluginCommand) Synopsis() string { - return "internal - lanch a syslog collector plugin" + return "internal - launch a syslog collector plugin" } func (s *SyslogPluginCommand) Run(args []string) int { - if len(args) == 0 { - s.Ui.Error("log output file isn't provided") + if len(args) == 2 { + s.Ui.Error("log output file and log level are not provided") return 1 } logFileName := args[0] + logLevel := args[1] stdo, err := os.OpenFile(logFileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) if err != nil { s.Ui.Error(err.Error()) @@ -37,7 +38,7 @@ func (s *SyslogPluginCommand) Run(args []string) int { } plugin.Serve(&plugin.ServeConfig{ HandshakeConfig: driver.HandshakeConfig, - Plugins: driver.GetPluginMap(stdo), + Plugins: driver.GetPluginMap(stdo, logLevel), }) return 0 From dc8a940f9c45965325c4addc627fe061e4736037 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Mon, 9 Jan 2017 14:07:17 -0800 Subject: [PATCH 2/3] Remove unused code --- command/syslog_plugin.go | 45 ---------------------------------------- commands.go | 5 ----- 2 files changed, 50 deletions(-) delete mode 100644 command/syslog_plugin.go diff --git a/command/syslog_plugin.go b/command/syslog_plugin.go deleted file mode 100644 index 6b753ac15ce..00000000000 --- a/command/syslog_plugin.go +++ /dev/null @@ -1,45 +0,0 @@ -package command - -import ( - "os" - "strings" - - "github.com/hashicorp/go-plugin" - - "github.com/hashicorp/nomad/client/driver" -) - -type SyslogPluginCommand struct { - Meta -} - -func (e *SyslogPluginCommand) Help() string { - helpText := ` - This is a command used by Nomad internally to launch a syslog collector" - ` - return strings.TrimSpace(helpText) -} - -func (s *SyslogPluginCommand) Synopsis() string { - return "internal - launch a syslog collector plugin" -} - -func (s *SyslogPluginCommand) Run(args []string) int { - if len(args) == 2 { - s.Ui.Error("log output file and log level are not provided") - return 1 - } - logFileName := args[0] - logLevel := args[1] - stdo, err := os.OpenFile(logFileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) - if err != nil { - s.Ui.Error(err.Error()) - return 1 - } - plugin.Serve(&plugin.ServeConfig{ - HandshakeConfig: driver.HandshakeConfig, - Plugins: driver.GetPluginMap(stdo, logLevel), - }) - - return 0 -} diff --git a/commands.go b/commands.go index 08bad16eaee..3def861c439 100644 --- a/commands.go +++ b/commands.go @@ -126,11 +126,6 @@ func Commands(metaPtr *command.Meta) map[string]cli.CommandFactory { Meta: meta, }, nil }, - "syslog": func() (cli.Command, error) { - return &command.SyslogPluginCommand{ - Meta: meta, - }, nil - }, "server-force-leave": func() (cli.Command, error) { return &command.ServerForceLeaveCommand{ Meta: meta, From a066d90313c6946b04e10c8feed7b15a40c9a1f5 Mon Sep 17 00:00:00 2001 From: Diptanu Choudhury Date: Thu, 12 Jan 2017 11:50:49 -0800 Subject: [PATCH 3/3] Added executorconfig --- client/driver/docker.go | 15 +++---- client/driver/driver.go | 10 ++--- client/driver/exec.go | 16 +++----- client/driver/java.go | 15 +++---- client/driver/plugins.go | 10 ++--- client/driver/qemu.go | 15 +++---- client/driver/raw_exec.go | 15 +++---- client/driver/rkt.go | 15 +++---- client/driver/structs/structs.go | 10 +++++ client/driver/syslog_plugin.go | 69 -------------------------------- client/driver/utils.go | 40 +++++++++++++++++- client/task_runner.go | 4 +- command/executor_plugin.go | 17 +++++--- 13 files changed, 97 insertions(+), 154 deletions(-) delete mode 100644 client/driver/syslog_plugin.go diff --git a/client/driver/docker.go b/client/driver/docker.go index 79db02673c1..a588848125d 100644 --- a/client/driver/docker.go +++ b/client/driver/docker.go @@ -6,7 +6,6 @@ import ( "log" "net" "os" - "os/exec" "path/filepath" "regexp" "runtime" @@ -26,7 +25,6 @@ import ( "github.com/hashicorp/nomad/client/driver/executor" dstructs "github.com/hashicorp/nomad/client/driver/structs" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/helper/fields" shelpers "github.com/hashicorp/nomad/helper/stats" "github.com/hashicorp/nomad/nomad/structs" @@ -387,17 +385,14 @@ func (d *DockerDriver) Prestart(ctx *ExecContext, task *structs.Task) error { } func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { - bin, err := discover.NomadExecutable() - if err != nil { - return nil, fmt.Errorf("unable to find the nomad binary: %v", err) - } pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") - pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), + executorConfig := &dstructs.ExecutorConfig{ + LogFile: pluginLogFile, + LogLevel: d.config.LogLevel, } - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) if err != nil { return nil, err } @@ -1119,7 +1114,7 @@ func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, er if !found { return nil, fmt.Errorf("Failed to find container %s", pid.ContainerID) } - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) if err != nil { d.logger.Printf("[INFO] driver.docker: couldn't re-attach to the plugin process: %v", err) d.logger.Printf("[DEBUG] driver.docker: stopping container %q", pid.ContainerID) diff --git a/client/driver/driver.go b/client/driver/driver.go index 97db2d1d4ec..c4560e9781d 100644 --- a/client/driver/driver.go +++ b/client/driver/driver.go @@ -152,17 +152,13 @@ type ExecContext struct { // Alloc ID AllocID string - - // LogLevel is the level of the logs to putout - LogLevel string } // NewExecContext is used to create a new execution context -func NewExecContext(td *allocdir.TaskDir, allocID string, logLevel string) *ExecContext { +func NewExecContext(td *allocdir.TaskDir, allocID string) *ExecContext { return &ExecContext{ - TaskDir: td, - AllocID: allocID, - LogLevel: logLevel, + TaskDir: td, + AllocID: allocID, } } diff --git a/client/driver/exec.go b/client/driver/exec.go index 93e68cd1fa9..3d1930e169d 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "os" - "os/exec" "path/filepath" "time" @@ -15,7 +14,6 @@ import ( "github.com/hashicorp/nomad/client/driver/executor" dstructs "github.com/hashicorp/nomad/client/driver/structs" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/helper/fields" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" @@ -110,16 +108,12 @@ func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, err } - bin, err := discover.NomadExecutable() - if err != nil { - return nil, fmt.Errorf("unable to find the nomad binary: %v", err) - } pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") - pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), + executorConfig := &dstructs.ExecutorConfig{ + LogFile: pluginLogFile, + LogLevel: d.config.LogLevel, } - - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) if err != nil { return nil, err } @@ -191,7 +185,7 @@ func (d *ExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro pluginConfig := &plugin.ClientConfig{ Reattach: id.PluginConfig.PluginConfig(), } - exec, client, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, client, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) if err != nil { merrs := new(multierror.Error) merrs.Errors = append(merrs.Errors, err) diff --git a/client/driver/java.go b/client/driver/java.go index 4ce6b109339..4519a719aac 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -22,7 +22,6 @@ import ( dstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/fingerprint" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/helper/fields" "github.com/hashicorp/nomad/nomad/structs" ) @@ -191,17 +190,13 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, args = append(args, driverConfig.Args...) } - bin, err := discover.NomadExecutable() - if err != nil { - return nil, fmt.Errorf("unable to find the nomad binary: %v", err) - } - pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") - pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), + executorConfig := &dstructs.ExecutorConfig{ + LogFile: pluginLogFile, + LogLevel: d.config.LogLevel, } - execIntf, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + execIntf, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) if err != nil { return nil, err } @@ -285,7 +280,7 @@ func (d *JavaDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro pluginConfig := &plugin.ClientConfig{ Reattach: id.PluginConfig.PluginConfig(), } - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) if err != nil { merrs := new(multierror.Error) merrs.Errors = append(merrs.Errors, err) diff --git a/client/driver/plugins.go b/client/driver/plugins.go index 8818611b566..cd9522c1f4e 100644 --- a/client/driver/plugins.go +++ b/client/driver/plugins.go @@ -2,7 +2,6 @@ package driver import ( "io" - "io/ioutil" "log" "net" "strings" @@ -22,16 +21,13 @@ func GetPluginMap(w io.Writer, logLevel string) map[string]plugin.Plugin { filter := &logutils.LevelFilter{ Levels: []logutils.LogLevel{"TRACE", "DEBUG", "INFO", "WARN", "ERR"}, MinLevel: logutils.LogLevel(strings.ToUpper(logLevel)), - Writer: ioutil.Discard, + Writer: w, } - e.logger = log.New(filter, "", log.LstdFlags) + e.logger = log.New(filter, "", log.LstdFlags|log.Lmicroseconds) - s := new(SyslogCollectorPlugin) - s.logger = log.New(w, "", log.LstdFlags) return map[string]plugin.Plugin{ - "executor": e, - "syslogcollector": s, + "executor": e, } } diff --git a/client/driver/qemu.go b/client/driver/qemu.go index f2fc0eee40b..98575c5e635 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -18,7 +18,6 @@ import ( dstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/fingerprint" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/helper/fields" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" @@ -233,17 +232,13 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, } d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " ")) - bin, err := discover.NomadExecutable() - if err != nil { - return nil, fmt.Errorf("unable to find the nomad binary: %v", err) - } - pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") - pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), + executorConfig := &dstructs.ExecutorConfig{ + LogFile: pluginLogFile, + LogLevel: d.config.LogLevel, } - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) if err != nil { return nil, err } @@ -311,7 +306,7 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro Reattach: id.PluginConfig.PluginConfig(), } - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) if err != nil { d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid") if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index 942a9d57e6c..eb4e1b7900e 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -5,7 +5,6 @@ import ( "fmt" "log" "os" - "os/exec" "path/filepath" "time" @@ -15,7 +14,6 @@ import ( dstructs "github.com/hashicorp/nomad/client/driver/structs" "github.com/hashicorp/nomad/client/fingerprint" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/helper/fields" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" @@ -124,16 +122,13 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl return nil, err } - bin, err := discover.NomadExecutable() - if err != nil { - return nil, fmt.Errorf("unable to find the nomad binary: %v", err) - } pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") - pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), + executorConfig := &dstructs.ExecutorConfig{ + LogFile: pluginLogFile, + LogLevel: d.config.LogLevel, } - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) if err != nil { return nil, err } @@ -199,7 +194,7 @@ func (d *RawExecDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, e pluginConfig := &plugin.ClientConfig{ Reattach: id.PluginConfig.PluginConfig(), } - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) if err != nil { d.logger.Println("[ERR] driver.raw_exec: error connecting to plugin so destroying plugin pid and user pid") if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { diff --git a/client/driver/rkt.go b/client/driver/rkt.go index 3fa1be50b51..d9962136895 100644 --- a/client/driver/rkt.go +++ b/client/driver/rkt.go @@ -23,7 +23,6 @@ import ( "github.com/hashicorp/nomad/client/driver/executor" dstructs "github.com/hashicorp/nomad/client/driver/structs" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/helper/fields" "github.com/hashicorp/nomad/nomad/structs" "github.com/mitchellh/mapstructure" @@ -394,17 +393,13 @@ func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, e filter := strings.Split(d.config.ReadDefault("env.blacklist", config.DefaultEnvBlacklist), ",") d.taskEnv.AppendHostEnvvars(filter) - bin, err := discover.NomadExecutable() - if err != nil { - return nil, fmt.Errorf("unable to find the nomad binary: %v", err) - } - pluginLogFile := filepath.Join(ctx.TaskDir.Dir, fmt.Sprintf("%s-executor.out", task.Name)) - pluginConfig := &plugin.ClientConfig{ - Cmd: exec.Command(bin, "executor", pluginLogFile, ctx.LogLevel), + executorConfig := &dstructs.ExecutorConfig{ + LogFile: pluginLogFile, + LogLevel: d.config.LogLevel, } - execIntf, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + execIntf, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) if err != nil { return nil, err } @@ -467,7 +462,7 @@ func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error pluginConfig := &plugin.ClientConfig{ Reattach: id.PluginConfig.PluginConfig(), } - exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) + exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) if err != nil { d.logger.Println("[ERROR] driver.rkt: error connecting to plugin so destroying plugin pid and user pid") if e := destroyPlugin(id.PluginConfig.Pid, id.ExecutorPid); e != nil { diff --git a/client/driver/structs/structs.go b/client/driver/structs/structs.go index 0fa67ff2c06..aed1e831ad7 100644 --- a/client/driver/structs/structs.go +++ b/client/driver/structs/structs.go @@ -55,3 +55,13 @@ type CheckResult struct { // Err is the error that a check returned Err error } + +// ExecutorConfig is the config that Nomad passes to the executor +type ExecutorConfig struct { + + // LogFile is the file to which Executor logs + LogFile string + + // LogLevel is the level of the logs to putout + LogLevel string +} diff --git a/client/driver/syslog_plugin.go b/client/driver/syslog_plugin.go deleted file mode 100644 index 55237cd2d2f..00000000000 --- a/client/driver/syslog_plugin.go +++ /dev/null @@ -1,69 +0,0 @@ -package driver - -import ( - "log" - "net/rpc" - - "github.com/hashicorp/go-plugin" - "github.com/hashicorp/nomad/client/driver/logging" - "github.com/hashicorp/nomad/nomad/structs" -) - -type SyslogCollectorRPC struct { - client *rpc.Client -} - -type LaunchCollectorArgs struct { - Ctx *logging.LogCollectorContext -} - -func (e *SyslogCollectorRPC) LaunchCollector(ctx *logging.LogCollectorContext) (*logging.SyslogCollectorState, error) { - var ss *logging.SyslogCollectorState - err := e.client.Call("Plugin.LaunchCollector", LaunchCollectorArgs{Ctx: ctx}, &ss) - return ss, err -} - -func (e *SyslogCollectorRPC) Exit() error { - return e.client.Call("Plugin.Exit", new(interface{}), new(interface{})) -} - -func (e *SyslogCollectorRPC) UpdateLogConfig(logConfig *structs.LogConfig) error { - return e.client.Call("Plugin.UpdateLogConfig", logConfig, new(interface{})) -} - -type SyslogCollectorRPCServer struct { - Impl logging.LogCollector -} - -func (s *SyslogCollectorRPCServer) LaunchCollector(args LaunchCollectorArgs, - resp *logging.SyslogCollectorState) error { - ss, err := s.Impl.LaunchCollector(args.Ctx) - if ss != nil { - *resp = *ss - } - return err -} - -func (s *SyslogCollectorRPCServer) Exit(args interface{}, resp *interface{}) error { - return s.Impl.Exit() -} - -func (s *SyslogCollectorRPCServer) UpdateLogConfig(logConfig *structs.LogConfig, resp *interface{}) error { - return s.Impl.UpdateLogConfig(logConfig) -} - -type SyslogCollectorPlugin struct { - logger *log.Logger - Impl *SyslogCollectorRPCServer -} - -func (p *SyslogCollectorPlugin) Server(*plugin.MuxBroker) (interface{}, error) { - if p.Impl == nil { - p.Impl = &SyslogCollectorRPCServer{Impl: logging.NewSyslogCollector(p.logger)} - } - return p.Impl, nil -} - -func (p *SyslogCollectorPlugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) { - return &SyslogCollectorRPC{client: c}, nil -} diff --git a/client/driver/utils.go b/client/driver/utils.go index 9fc465bb391..51de500a8b5 100644 --- a/client/driver/utils.go +++ b/client/driver/utils.go @@ -1,6 +1,7 @@ package driver import ( + "encoding/json" "fmt" "io" "os" @@ -14,13 +15,27 @@ import ( "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/executor" cstructs "github.com/hashicorp/nomad/client/driver/structs" + "github.com/hashicorp/nomad/helper/discover" "github.com/hashicorp/nomad/nomad/structs" ) // createExecutor launches an executor plugin and returns an instance of the // Executor interface -func createExecutor(config *plugin.ClientConfig, w io.Writer, - clientConfig *config.Config) (executor.Executor, *plugin.Client, error) { +func createExecutor(w io.Writer, clientConfig *config.Config, + executorConfig *cstructs.ExecutorConfig) (executor.Executor, *plugin.Client, error) { + + c, err := json.Marshal(executorConfig) + if err != nil { + return nil, nil, fmt.Errorf("unable to create executor config: %v", err) + } + bin, err := discover.NomadExecutable() + if err != nil { + return nil, nil, fmt.Errorf("unable to find the nomad binary: %v", err) + } + + config := &plugin.ClientConfig{ + Cmd: exec.Command(bin, "executor", string(c)), + } config.HandshakeConfig = HandshakeConfig config.Plugins = GetPluginMap(w, clientConfig.LogLevel) config.MaxPort = clientConfig.ClientMaxPort @@ -46,6 +61,27 @@ func createExecutor(config *plugin.ClientConfig, w io.Writer, return executorPlugin, executorClient, nil } +func createExecutorWithConfig(config *plugin.ClientConfig, w io.Writer) (executor.Executor, *plugin.Client, error) { + config.HandshakeConfig = HandshakeConfig + + // Setting this to DEBUG since the log level at the executor server process + // is already set, and this effects only the executor client. + config.Plugins = GetPluginMap(w, "DEBUG") + + executorClient := plugin.NewClient(config) + rpcClient, err := executorClient.Client() + if err != nil { + return nil, nil, fmt.Errorf("error creating rpc client for executor plugin: %v", err) + } + + raw, err := rpcClient.Dispense("executor") + if err != nil { + return nil, nil, fmt.Errorf("unable to dispense the executor plugin: %v", err) + } + executorPlugin := raw.(executor.Executor) + return executorPlugin, executorClient, nil +} + func consulContext(clientConfig *config.Config, containerID string) *executor.ConsulContext { return &executor.ConsulContext{ ConsulConfig: clientConfig.ConsulConfig, diff --git a/client/task_runner.go b/client/task_runner.go index 48f9c6207d6..f8331f80384 100644 --- a/client/task_runner.go +++ b/client/task_runner.go @@ -267,7 +267,7 @@ func (r *TaskRunner) RestoreState() error { return err } - ctx := driver.NewExecContext(r.taskDir, r.alloc.ID, r.config.LogLevel) + ctx := driver.NewExecContext(r.taskDir, r.alloc.ID) handle, err := d.Open(ctx, snap.HandleID) // In the case it fails, we relaunch the task in the Run() method. @@ -1094,7 +1094,7 @@ func (r *TaskRunner) startTask() error { } // Run prestart - ctx := driver.NewExecContext(r.taskDir, r.alloc.ID, r.config.LogLevel) + ctx := driver.NewExecContext(r.taskDir, r.alloc.ID) if err := drv.Prestart(ctx, r.task); err != nil { wrapped := fmt.Errorf("failed to initialize task %q for alloc %q: %v", r.task.Name, r.alloc.ID, err) diff --git a/command/executor_plugin.go b/command/executor_plugin.go index 24a32853419..93da6e1b5af 100644 --- a/command/executor_plugin.go +++ b/command/executor_plugin.go @@ -1,12 +1,14 @@ package command import ( + "encoding/json" "os" "strings" "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/client/driver" + dstructs "github.com/hashicorp/nomad/client/driver/structs" ) type ExecutorPluginCommand struct { @@ -25,20 +27,23 @@ func (e *ExecutorPluginCommand) Synopsis() string { } func (e *ExecutorPluginCommand) Run(args []string) int { - if len(args) != 2 { - e.Ui.Error("log output file and log level are not provided") + if len(args) != 1 { + e.Ui.Error("json configuration not provided") return 1 } - logFileName := args[0] - logLevel := args[1] - stdo, err := os.OpenFile(logFileName, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) + config := args[0] + var executorConfig dstructs.ExecutorConfig + if err := json.Unmarshal([]byte(config), &executorConfig); err != nil { + return 1 + } + stdo, err := os.OpenFile(executorConfig.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) if err != nil { e.Ui.Error(err.Error()) return 1 } plugin.Serve(&plugin.ServeConfig{ HandshakeConfig: driver.HandshakeConfig, - Plugins: driver.GetPluginMap(stdo, logLevel), + Plugins: driver.GetPluginMap(stdo, executorConfig.LogLevel), }) return 0 }