Add envconsul-like support and refactor environment handling #2654
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,10 @@ | ||
| package client | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "bytes" | ||
| "fmt" | ||
| "io" | ||
| "math/rand" | ||
| "os" | ||
| "path/filepath" | ||
|
|
@@ -76,7 +79,7 @@ type TaskTemplateManager struct { | |
|
|
||
| func NewTaskTemplateManager(hook TaskHooks, tmpls []*structs.Template, | ||
| config *config.Config, vaultToken, taskDir string, | ||
| envBuilder *env.Builder) (*TaskTemplateManager, error) { | ||
|
|
||
| // Check pre-conditions | ||
| if hook == nil { | ||
|
|
@@ -85,7 +88,7 @@ func NewTaskTemplateManager(hook TaskHooks, tmpls []*structs.Template, | |
| return nil, fmt.Errorf("Invalid config given") | ||
| } else if taskDir == "" { | ||
| return nil, fmt.Errorf("Invalid task directory given") | ||
| } else if envBuilder == nil { | ||
| return nil, fmt.Errorf("Invalid task environment given") | ||
| } | ||
|
|
||
|
|
@@ -114,14 +117,14 @@ func NewTaskTemplateManager(hook TaskHooks, tmpls []*structs.Template, | |
| } | ||
|
|
||
| // Build the consul-template runner | ||
| runner, lookup, err := templateRunner(tmpls, config, vaultToken, taskDir, envBuilder.Build()) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| tm.runner = runner | ||
| tm.lookup = lookup | ||
|
|
||
| go tm.run(envBuilder, taskDir) | ||
| return tm, nil | ||
| } | ||
|
|
||
|
|
@@ -144,7 +147,7 @@ func (tm *TaskTemplateManager) Stop() { | |
| } | ||
|
|
||
| // run is the long lived loop that handles errors and templates being rendered | ||
| func (tm *TaskTemplateManager) run(envBuilder *env.Builder, taskDir string) { | ||
| // Runner is nil if there is no templates | ||
| if tm.runner == nil { | ||
| // Unblock the start if there is nothing to do | ||
|
|
@@ -192,6 +195,12 @@ WAIT: | |
| } | ||
| } | ||
|
|
||
| for _, t := range tm.templates { | ||
|
There was a problem hiding this comment. Comment explaining what you are doing. Blank line between the block please |
||
| if err := loadTemplateEnv(envBuilder, taskDir, t); err != nil { | ||
| tm.hook.Kill("consul-template", err.Error(), true) | ||
| return | ||
| } | ||
| } | ||
| allRenderedTime = time.Now() | ||
| tm.hook.UnblockStart("consul-template") | ||
|
|
||
|
|
@@ -243,6 +252,11 @@ WAIT: | |
| } | ||
|
|
||
| for _, tmpl := range tmpls { | ||
| if err := loadTemplateEnv(envBuilder, taskDir, tmpl); err != nil { | ||
|
|
||
|
|
||
| tm.hook.Kill("consul-template", err.Error(), true) | ||
| return | ||
| } | ||
|
|
||
| switch tmpl.ChangeMode { | ||
| case structs.TemplateChangeModeSignal: | ||
| signals[tmpl.ChangeSignal] = struct{}{} | ||
|
|
@@ -317,7 +331,7 @@ func (tm *TaskTemplateManager) allTemplatesNoop() bool { | |
| // lookup by destination to the template. If no templates are given, a nil | ||
| // template runner and lookup is returned. | ||
| func templateRunner(tmpls []*structs.Template, config *config.Config, | ||
| vaultToken, taskDir string, taskEnv *env.TaskEnv) ( | ||
| *manager.Runner, map[string][]*structs.Template, error) { | ||
|
|
||
| if len(tmpls) == 0 { | ||
|
|
@@ -350,7 +364,7 @@ func templateRunner(tmpls []*structs.Template, config *config.Config, | |
| } | ||
|
|
||
| // Set Nomad's environment variables | ||
| runner.Env = taskEnv.All() | ||
|
|
||
| // Build the lookup | ||
| idMap := runner.TemplateConfigMapping() | ||
|
|
@@ -368,9 +382,7 @@ func templateRunner(tmpls []*structs.Template, config *config.Config, | |
|
|
||
| // parseTemplateConfigs converts the tasks templates into consul-templates | ||
| func parseTemplateConfigs(tmpls []*structs.Template, taskDir string, | ||
| taskEnv *env.TaskEnv, allowAbs bool) (map[ctconf.TemplateConfig]*structs.Template, error) { | ||
|
|
||
| ctmpls := make(map[ctconf.TemplateConfig]*structs.Template, len(tmpls)) | ||
| for _, tmpl := range tmpls { | ||
|
|
@@ -492,3 +504,60 @@ func runnerConfig(config *config.Config, vaultToken string) (*ctconf.Config, err | |
| conf.Finalize() | ||
| return conf, nil | ||
| } | ||
|
|
||
| // loadTemplateEnv loads task environment variables from templates. | ||
| func loadTemplateEnv(builder *env.Builder, taskDir string, t *structs.Template) error { | ||
| if !t.Envvars { | ||
| return nil | ||
| } | ||
| f, err := os.Open(filepath.Join(taskDir, t.DestPath)) | ||
| if err != nil { | ||
| return fmt.Errorf("error opening env template: %v", err) | ||
| } | ||
| defer f.Close() | ||
|
|
||
| // Parse environment fil | ||
| vars, err := parseEnvFile(f) | ||
| if err != nil { | ||
| return fmt.Errorf("error parsing env template %q: %v", t.DestPath, err) | ||
| } | ||
|
|
||
| // Set the environment variables | ||
| builder.SetTemplateEnv(vars) | ||
|
There was a problem hiding this comment. This breaks if they have multiple templates generating environment variables. There was a problem hiding this comment. Wow, good catch. Will add a test as well. |
||
| return nil | ||
| } | ||
|
|
||
| // parseEnvFile and return a map of the environment variables suitable for | ||
| // TaskEnvironment.AppendEnvvars or an error. | ||
| // | ||
| // See nomad/structs#Template.Envvars comment for format. | ||
| func parseEnvFile(r io.Reader) (map[string]string, error) { | ||
|
There was a problem hiding this comment. It may be worth just using this: https://github.com/joho/godotenv There was a problem hiding this comment. Lots of edge cases we don't need to handle There was a problem hiding this comment. Since this is tied to a template it must be a single file. It's probably worth documenting how to use env files from artifacts by setting source path to where the artifact is downloaded, but env dirs (or autoloading |
||
| vars := make(map[string]string, 50) | ||
| lines := 0 | ||
| scanner := bufio.NewScanner(r) | ||
| for scanner.Scan() { | ||
| lines++ | ||
| buf := scanner.Bytes() | ||
| if len(buf) == 0 { | ||
| // Skip empty lines | ||
| continue | ||
| } | ||
| if buf[0] == '#' { | ||
| // Skip lines starting with a # | ||
| continue | ||
| } | ||
| n := bytes.IndexByte(buf, '=') | ||
| if n == -1 { | ||
| return nil, fmt.Errorf("line %d: no '=' sign: %q", lines, string(buf)) | ||
| } | ||
| if len(buf) > n { | ||
| vars[string(buf[0:n])] = string(buf[n+1 : len(buf)]) | ||
| } else { | ||
| vars[string(buf[0:n])] = "" | ||
| } | ||
| } | ||
| if err := scanner.Err(); err != nil { | ||
| return nil, err | ||
| } | ||
| return vars, nil | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Believe there is a canonicalize test as well