From a736c82015c391e852432c80941e5066d29b0992 Mon Sep 17 00:00:00 2001 From: Jeroen Op 't Eynde Date: Sun, 10 Jan 2021 12:52:09 +0100 Subject: [PATCH] feat: multiple inline environments (#476) * feat(api): Loader.List Introduces `Loader.List(path, opts)` which returns all environments that could possibly be found at path. This can be used to prompt the user for the correct inline environment, while the actual Load does not need to be concerned with this * feat: list multiple inline environments * feat(cli): --name to select environment * s/FindEnvs/ListEnvs * find multiple inline environments in parallel * LoaderOpts * FindEnvs by name * remove Println * Revert "FindEnvs by name" This reverts commit e6dd7e7cf4292b3af01036cb713d79c2d7c46193. * review comments Co-authored-by: sh0rez --- cmd/tk/args.go | 2 +- cmd/tk/env.go | 18 ++----- cmd/tk/export.go | 21 ++++++-- cmd/tk/flags.go | 2 + cmd/tk/workflow.go | 4 ++ pkg/tanka/errors.go | 2 +- pkg/tanka/evaluators.go | 66 ++++++++++++++++++++++++- pkg/tanka/export.go | 13 +---- pkg/tanka/{list.go => find.go} | 56 ++++++++++++--------- pkg/tanka/inline.go | 90 ++++++++++++++++++++++++++-------- pkg/tanka/load.go | 33 ++++++++++--- pkg/tanka/parallel.go | 25 ++++++++-- pkg/tanka/static.go | 17 +++++-- pkg/tanka/tanka.go | 3 ++ 14 files changed, 262 insertions(+), 90 deletions(-) rename pkg/tanka/{list.go => find.go} (54%) diff --git a/cmd/tk/args.go b/cmd/tk/args.go index e496ff9d5..ddf943d39 100644 --- a/cmd/tk/args.go +++ b/cmd/tk/args.go @@ -24,7 +24,7 @@ var workflowArgs = cli.Args{ return nil } - envs, err := tanka.ListEnvs(pwd, tanka.ListOpts{}) + envs, err := tanka.FindEnvs(pwd, tanka.FindOpts{}) if err != nil { return nil } diff --git a/cmd/tk/env.go b/cmd/tk/env.go index e69691841..53b572fb5 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -77,7 +77,7 @@ func envSetCmd() *cli.Command { tmp.Spec.APIServer = server } - cfg, err := tanka.Peek(path, tanka.JsonnetOpts{}) + cfg, err := tanka.Peek(path, tanka.Opts{}) if err != nil { return err } @@ -229,26 +229,18 @@ func envListCmd() *cli.Command { useNames := cmd.Flags().Bool("names", false, "plain names output") cmd.Run = func(cmd *cli.Command, args []string) error { - var dir string + var path string var err error if len(args) == 1 { - dir = args[0] + path = args[0] } else { - dir, err = os.Getwd() + path, err = os.Getwd() if err != nil { return nil } } - stat, err := os.Stat(dir) - if err != nil { - return err - } - if !stat.IsDir() { - return fmt.Errorf("Not a directory: %s", dir) - } - - envs, err := tanka.ListEnvs(dir, tanka.ListOpts{Selector: getLabelSelector()}) + envs, err := tanka.FindEnvs(path, tanka.FindOpts{Selector: getLabelSelector()}) if err != nil { return err } diff --git a/cmd/tk/export.go b/cmd/tk/export.go index dddc77059..bb55fda32 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/grafana/tanka/pkg/jsonnet/jpath" + "github.com/grafana/tanka/pkg/process" "github.com/grafana/tanka/pkg/tanka" ) @@ -43,13 +44,19 @@ func exportCmd() *cli.Command { recursive := cmd.Flags().BoolP("recursive", "r", false, "Look recursively for Tanka environments") cmd.Run = func(cmd *cli.Command, args []string) error { + filters, err := process.StrExps(vars.targets...) + if err != nil { + return err + } + opts := tanka.ExportEnvOpts{ Format: *format, Extension: *extension, Merge: *merge, - Targets: vars.targets, Opts: tanka.Opts{ JsonnetOpts: getJsonnetOpts(), + Filters: filters, + Name: vars.name, }, Selector: getLabelSelector(), Parallelism: *parallel, @@ -65,7 +72,7 @@ func exportCmd() *cli.Command { } // get absolute path to Environment - envs, err := tanka.ListEnvs(path, tanka.ListOpts{Selector: opts.Selector}) + envs, err := tanka.FindEnvs(path, tanka.FindOpts{Selector: opts.Selector}) if err != nil { return err } @@ -77,8 +84,14 @@ func exportCmd() *cli.Command { } // validate environment - if _, err := tanka.Peek(path, opts.Opts.JsonnetOpts); err != nil { - return err + if _, err := tanka.Peek(path, opts.Opts); err != nil { + switch err.(type) { + case tanka.ErrMultipleEnvs: + fmt.Println("Please use --name to export a single environment or --recursive to export multiple environments.") + return err + default: + return err + } } paths = append(paths, path) diff --git a/cmd/tk/flags.go b/cmd/tk/flags.go index aab5af384..96a005476 100644 --- a/cmd/tk/flags.go +++ b/cmd/tk/flags.go @@ -13,11 +13,13 @@ import ( ) type workflowFlagVars struct { + name string targets []string } func workflowFlags(fs *pflag.FlagSet) *workflowFlagVars { v := workflowFlagVars{} + fs.StringVar(&v.name, "name", "", "selects an environment from inline environments") fs.StringSliceVarP(&v.targets, "target", "t", nil, "only use the specified objects (Format: /)") return &v } diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go index c66c31ab4..9d3f73cbf 100644 --- a/cmd/tk/workflow.go +++ b/cmd/tk/workflow.go @@ -43,6 +43,7 @@ func applyCmd() *cli.Command { } opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() + opts.Name = vars.name return tanka.Apply(args[0], opts) } @@ -92,6 +93,7 @@ func deleteCmd() *cli.Command { } opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() + opts.Name = vars.name return tanka.Delete(args[0], opts) } @@ -122,6 +124,7 @@ func diffCmd() *cli.Command { } opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() + opts.Name = vars.name changes, err := tanka.Diff(args[0], opts) if err != nil { @@ -173,6 +176,7 @@ Otherwise run tk show --dangerous-allow-redirect to bypass this check.`) pretty, err := tanka.Show(args[0], tanka.Opts{ JsonnetOpts: getJsonnetOpts(), Filters: filters, + Name: vars.name, }) if err != nil { diff --git a/pkg/tanka/errors.go b/pkg/tanka/errors.go index 8422b55a0..a8ae5398e 100644 --- a/pkg/tanka/errors.go +++ b/pkg/tanka/errors.go @@ -22,7 +22,7 @@ type ErrMultipleEnvs struct { } func (e ErrMultipleEnvs) Error() string { - return fmt.Sprintf("found multiple Environments (%s) in '%s'", strings.Join(e.names, ", "), e.path) + return fmt.Sprintf("found multiple Environments in '%s': \n - %s", e.path, strings.Join(e.names, "\n - ")) } // ErrParallel is an array of errors collected while parsing environments in parallel diff --git a/pkg/tanka/evaluators.go b/pkg/tanka/evaluators.go index b8f145090..e262922e1 100644 --- a/pkg/tanka/evaluators.go +++ b/pkg/tanka/evaluators.go @@ -53,8 +53,8 @@ function(%s) const PatternEvalScript = "main.%s" -// EnvsOnlyEvalScript finds the Environment object (without its .data object) -const EnvsOnlyEvalScript = ` +// MetadataEvalScript finds the Environment object (without its .data object) +const MetadataEvalScript = ` local noDataEnv(object) = std.prune( if std.isObject(object) @@ -83,3 +83,65 @@ local noDataEnv(object) = noDataEnv(main) ` + +// MetadataSingleEnvEvalScript returns a Single Environment object +const MetadataSingleEnvEvalScript = ` +local singleEnv(object) = + std.prune( + if std.isObject(object) + then + if std.objectHas(object, 'apiVersion') + && std.objectHas(object, 'kind') + then + if object.kind == 'Environment' + && object.metadata.name == '%s' + then object { data:: {} } + else {} + else + std.mapWithKey( + function(key, obj) + singleEnv(obj), + object + ) + else if std.isArray(object) + then + std.map( + function(obj) + singleEnv(obj), + object + ) + else {} + ); + +singleEnv(main) +` + +// SingleEnvEvalScript returns a Single Environment object +const SingleEnvEvalScript = ` +local singleEnv(object) = + if std.isObject(object) + then + if std.objectHas(object, 'apiVersion') + && std.objectHas(object, 'kind') + then + if object.kind == 'Environment' + && object.metadata.name == '%s' + then object + else {} + else + std.mapWithKey( + function(key, obj) + singleEnv(obj), + object + ) + else if std.isArray(object) + then + std.map( + function(obj) + singleEnv(obj), + object + ) + else {}; + +singleEnv(main) +` diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 970577a85..5466a2721 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -15,7 +15,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "github.com/grafana/tanka/pkg/kubernetes/manifest" - "github.com/grafana/tanka/pkg/process" ) // BelRune is a string of the Ascii character BEL which made computers ring in ancient times @@ -36,8 +35,6 @@ type ExportEnvOpts struct { Extension string // merge export with existing directory Merge bool - // optional: only export specified Kubernetes manifests - Targets []string // optional: options to parse Jsonnet Opts Opts // optional: filter environments based on labels @@ -61,7 +58,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { // get all environments for paths envs, err := parallelLoadEnvironments(paths, parallelOpts{ - JsonnetOpts: opts.Opts.JsonnetOpts, + Opts: opts.Opts, Selector: opts.Selector, Parallelism: opts.Parallelism, }) @@ -70,14 +67,8 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } for _, env := range envs { - // select targets to export - filter, err := process.StrExps(opts.Targets...) - if err != nil { - return err - } - // get the manifests - loaded, err := LoadManifests(env, filter) + loaded, err := LoadManifests(env, opts.Opts.Filters) if err != nil { return err } diff --git a/pkg/tanka/list.go b/pkg/tanka/find.go similarity index 54% rename from pkg/tanka/list.go rename to pkg/tanka/find.go index 056f3a546..36d15dbcf 100644 --- a/pkg/tanka/list.go +++ b/pkg/tanka/find.go @@ -2,25 +2,25 @@ package tanka import ( "io/ioutil" + "os" "path/filepath" - "github.com/grafana/tanka/pkg/jsonnet" "github.com/grafana/tanka/pkg/spec/v1alpha1" "k8s.io/apimachinery/pkg/labels" ) -// ListOpts are optional arguments for ListEnvs -type ListOpts struct { +// FindOpts are optional arguments for FindEnvs +type FindOpts struct { Selector labels.Selector } -// ListEnvs returns metadata of all environments recursively found in 'dir'. +// FindEnvs returns metadata of all environments recursively found in 'path'. // Each directory is tested and included if it is a valid environment, either // static or inline. If a directory is a valid environment, its subdirectories // are not checked. -func ListEnvs(dir string, opts ListOpts) ([]*v1alpha1.Environment, error) { - // list all environments at dir - envs, err := list(dir) +func FindEnvs(path string, opts FindOpts) ([]*v1alpha1.Environment, error) { + // find all environments at dir + envs, err := find(path) if err != nil { return nil, err } @@ -41,33 +41,43 @@ func ListEnvs(dir string, opts ListOpts) ([]*v1alpha1.Environment, error) { return filtered, nil } -// list implements the actual functionality described at 'ListEnvs' -func list(dir string) ([]*v1alpha1.Environment, error) { - // list directory, also checks if dir - files, err := ioutil.ReadDir(dir) +// find implements the actual functionality described at 'FindEnvs' +func find(path string) ([]*v1alpha1.Environment, error) { + // try if this has envs + list, err := List(path, Opts{}) + if len(list) != 0 && err == nil { + // it has. don't search deeper + return list, nil + } + + stat, err := os.Stat(path) if err != nil { return nil, err } - // try if this is an env - env, err := Peek(dir, jsonnet.Opts{}) - if err == nil { - // it is one. don't search deeper - return []*v1alpha1.Environment{env}, nil + // if path is a file, don't search deeper + if !stat.IsDir() { + return nil, nil + } + + // list directory + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, err } // it's not one. Maybe subdirectories are? - ch := make(chan listOut) + ch := make(chan findOut) routines := 0 - // recursively list in parallel + // recursively find in parallel for _, fi := range files { if !fi.IsDir() { continue } routines++ - go listShim(filepath.Join(dir, fi.Name()), ch) + go findShim(filepath.Join(path, fi.Name()), ch) } // collect parallel results @@ -90,12 +100,12 @@ func list(dir string) ([]*v1alpha1.Environment, error) { return envs, nil } -type listOut struct { +type findOut struct { envs []*v1alpha1.Environment err error } -func listShim(dir string, ch chan listOut) { - envs, err := list(dir) - ch <- listOut{envs: envs, err: err} +func findShim(dir string, ch chan findOut) { + envs, err := find(dir) + ch <- findOut{envs: envs, err: err} } diff --git a/pkg/tanka/inline.go b/pkg/tanka/inline.go index 4137b4c6a..871e8e8e4 100644 --- a/pkg/tanka/inline.go +++ b/pkg/tanka/inline.go @@ -17,18 +17,12 @@ import ( // Kubernetes resources are expected at the `data` key of this very type type InlineLoader struct{} -func (i *InlineLoader) Load(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { - raw, err := EvalJsonnet(path, opts) - if err != nil { - return nil, err - } - - var data interface{} - if err := json.Unmarshal([]byte(raw), &data); err != nil { - return nil, err +func (i *InlineLoader) Load(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { + if opts.Name != "" { + opts.JsonnetOpts.EvalScript = fmt.Sprintf(SingleEnvEvalScript, opts.Name) } - envs, err := extractEnvs(data) + envs, err := inlineEval(path, opts.JsonnetOpts) if err != nil { return nil, err } @@ -45,38 +39,94 @@ func (i *InlineLoader) Load(path string, opts JsonnetOpts) (*v1alpha1.Environmen return nil, fmt.Errorf("Found no environments in '%s'", path) } - root, err := jpath.FindRoot(path) + // TODO: Re-serializing the entire env here. This is horribly inefficient + envData, err := json.Marshal(envs[0]) if err != nil { return nil, err } - file, err := jpath.Entrypoint(path) + env, err := inlineParse(path, envData) if err != nil { return nil, err } - namespace, err := filepath.Rel(root, file) + return env, nil +} + +func (i *InlineLoader) Peek(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { + opts.JsonnetOpts.EvalScript = MetadataEvalScript + if opts.Name != "" { + opts.JsonnetOpts.EvalScript = fmt.Sprintf(MetadataSingleEnvEvalScript, opts.Name) + } + return i.Load(path, opts) +} + +func (i *InlineLoader) List(path string, opts LoaderOpts) ([]*v1alpha1.Environment, error) { + opts.JsonnetOpts.EvalScript = MetadataEvalScript + list, err := inlineEval(path, opts.JsonnetOpts) if err != nil { return nil, err } - // TODO: Re-serializing the entire env here. This is horribly inefficient - envData, err := json.Marshal(envs[0]) + envs := make([]*v1alpha1.Environment, 0, len(list)) + for _, raw := range list { + data, err := json.Marshal(raw) + if err != nil { + return nil, err + } + + env, err := inlineParse(path, data) + if err != nil { + return nil, err + } + + envs = append(envs, env) + } + + return envs, nil +} + +func inlineEval(path string, opts JsonnetOpts) (manifest.List, error) { + raw, err := EvalJsonnet(path, opts) if err != nil { return nil, err } - env, err := spec.Parse(envData, namespace) + var data interface{} + if err := json.Unmarshal([]byte(raw), &data); err != nil { + return nil, err + } + + envs, err := extractEnvs(data) if err != nil { return nil, err } - return env, nil + return envs, nil } -func (i *InlineLoader) Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { - opts.EvalScript = EnvsOnlyEvalScript - return i.Load(path, opts) +func inlineParse(path string, data []byte) (*v1alpha1.Environment, error) { + root, err := jpath.FindRoot(path) + if err != nil { + return nil, err + } + + file, err := jpath.Entrypoint(path) + if err != nil { + return nil, err + } + + namespace, err := filepath.Rel(root, file) + if err != nil { + return nil, err + } + + env, err := spec.Parse(data, namespace) + if err != nil { + return nil, err + } + + return env, nil } // extractEnvs filters out any Environment manifests diff --git a/pkg/tanka/load.go b/pkg/tanka/load.go index 3a37598cc..fe0f55a45 100644 --- a/pkg/tanka/load.go +++ b/pkg/tanka/load.go @@ -36,7 +36,7 @@ func LoadEnvironment(path string, opts Opts) (*v1alpha1.Environment, error) { return nil, err } - env, err := loader.Load(path, opts.JsonnetOpts) + env, err := loader.Load(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) if err != nil { return nil, err } @@ -59,13 +59,25 @@ func LoadManifests(env *v1alpha1.Environment, filters process.Matchers) (*LoadRe // Peek loads the metadata of the environment at path. To get resources as well, // use Load -func Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { +func Peek(path string, opts Opts) (*v1alpha1.Environment, error) { loader, err := DetectLoader(path) if err != nil { return nil, err } - return loader.Peek(path, opts) + return loader.Peek(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) +} + +// List finds metadata of all environments at path that could possibly be +// loaded. List can be used to deal with multiple inline environments, by first +// listing them, choosing the right one and then only loading that one +func List(path string, opts Opts) ([]*v1alpha1.Environment, error) { + loader, err := DetectLoader(path) + if err != nil { + return nil, err + } + + return loader.List(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) } // DetectLoader detects whether the environment is inline or static and picks @@ -89,11 +101,20 @@ func DetectLoader(path string) (Loader, error) { // Loader is an abstraction over the process of loading Environments type Loader interface { - // Load the environment with path - Load(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) + // Load a single environment at path + Load(path string, opts LoaderOpts) (*v1alpha1.Environment, error) // Peek only loads metadata and omits the actual resources - Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) + Peek(path string, opts LoaderOpts) (*v1alpha1.Environment, error) + + // List returns metadata of all possible environments at path that can be + // loaded + List(path string, opts LoaderOpts) ([]*v1alpha1.Environment, error) +} + +type LoaderOpts struct { + JsonnetOpts + Name string } type LoadResult struct { diff --git a/pkg/tanka/parallel.go b/pkg/tanka/parallel.go index 0ba6e78c9..3ddbd6953 100644 --- a/pkg/tanka/parallel.go +++ b/pkg/tanka/parallel.go @@ -11,7 +11,7 @@ import ( const defaultParallelism = 8 type parallelOpts struct { - JsonnetOpts JsonnetOpts + Opts Selector labels.Selector Parallelism int } @@ -19,7 +19,20 @@ type parallelOpts struct { // parallelLoadEnvironments evaluates multiple environments in parallel func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.Environment, error) { jobsCh := make(chan parallelJob) - outCh := make(chan parallelOut, len(paths)) + list := make(map[string]string) + for _, path := range paths { + envs, err := FindEnvs(path, FindOpts{opts.Selector}) + if err != nil { + return nil, err + } + for _, env := range envs { + if opts.Name != "" && opts.Name != env.Metadata.Name { + continue + } + list[env.Metadata.Name] = path + } + } + outCh := make(chan parallelOut, len(list)) if opts.Parallelism <= 0 { opts.Parallelism = defaultParallelism @@ -29,17 +42,19 @@ func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.En go parallelWorker(jobsCh, outCh) } - for _, path := range paths { + for name, path := range list { + o := opts.Opts + o.Name = name jobsCh <- parallelJob{ path: path, - opts: Opts{JsonnetOpts: opts.JsonnetOpts}, + opts: o, } } close(jobsCh) var envs []*v1alpha1.Environment var errors []error - for i := 0; i < len(paths); i++ { + for i := 0; i < len(list); i++ { out := <-outCh if out.err != nil { errors = append(errors, out.err) diff --git a/pkg/tanka/static.go b/pkg/tanka/static.go index 445ca0f98..cadb2550c 100644 --- a/pkg/tanka/static.go +++ b/pkg/tanka/static.go @@ -12,13 +12,13 @@ import ( // Jsonnet is evaluated as normal type StaticLoader struct{} -func (s StaticLoader) Load(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { - config, err := Peek(path, opts) +func (s StaticLoader) Load(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { + config, err := s.Peek(path, opts) if err != nil { return nil, err } - data, err := EvalJsonnet(path, opts) + data, err := EvalJsonnet(path, opts.JsonnetOpts) if err != nil { return nil, err } @@ -30,7 +30,7 @@ func (s StaticLoader) Load(path string, opts JsonnetOpts) (*v1alpha1.Environment return config, nil } -func (s StaticLoader) Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { +func (s StaticLoader) Peek(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { config, err := parseStaticSpec(path) if err != nil { return nil, err @@ -39,6 +39,15 @@ func (s StaticLoader) Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment return config, nil } +func (s StaticLoader) List(path string, opts LoaderOpts) ([]*v1alpha1.Environment, error) { + env, err := s.Peek(path, opts) + if err != nil { + return nil, err + } + + return []*v1alpha1.Environment{env}, nil +} + // parseStaticSpec parses the `spec.json` of the environment and returns a // *kubernetes.Kubernetes from it func parseStaticSpec(path string) (*v1alpha1.Environment, error) { diff --git a/pkg/tanka/tanka.go b/pkg/tanka/tanka.go index 36ce69753..d7f91a088 100644 --- a/pkg/tanka/tanka.go +++ b/pkg/tanka/tanka.go @@ -21,6 +21,9 @@ type Opts struct { // Filters are used to optionally select a subset of the resources Filters process.Matchers + + // Name is used to extract a single environment from multiple environments + Name string } // DEFAULT_DEV_VERSION is the placeholder version used when no actual semver is