diff --git a/cmd/tk/args.go b/cmd/tk/args.go index e496ff9d5..ddf943d39 100644 --- a/cmd/tk/args.go +++ b/cmd/tk/args.go @@ -24,7 +24,7 @@ var workflowArgs = cli.Args{ return nil } - envs, err := tanka.ListEnvs(pwd, tanka.ListOpts{}) + envs, err := tanka.FindEnvs(pwd, tanka.FindOpts{}) if err != nil { return nil } diff --git a/cmd/tk/env.go b/cmd/tk/env.go index e69691841..53b572fb5 100644 --- a/cmd/tk/env.go +++ b/cmd/tk/env.go @@ -77,7 +77,7 @@ func envSetCmd() *cli.Command { tmp.Spec.APIServer = server } - cfg, err := tanka.Peek(path, tanka.JsonnetOpts{}) + cfg, err := tanka.Peek(path, tanka.Opts{}) if err != nil { return err } @@ -229,26 +229,18 @@ func envListCmd() *cli.Command { useNames := cmd.Flags().Bool("names", false, "plain names output") cmd.Run = func(cmd *cli.Command, args []string) error { - var dir string + var path string var err error if len(args) == 1 { - dir = args[0] + path = args[0] } else { - dir, err = os.Getwd() + path, err = os.Getwd() if err != nil { return nil } } - stat, err := os.Stat(dir) - if err != nil { - return err - } - if !stat.IsDir() { - return fmt.Errorf("Not a directory: %s", dir) - } - - envs, err := tanka.ListEnvs(dir, tanka.ListOpts{Selector: getLabelSelector()}) + envs, err := tanka.FindEnvs(path, tanka.FindOpts{Selector: getLabelSelector()}) if err != nil { return err } diff --git a/cmd/tk/export.go b/cmd/tk/export.go index dddc77059..bb55fda32 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/grafana/tanka/pkg/jsonnet/jpath" + "github.com/grafana/tanka/pkg/process" "github.com/grafana/tanka/pkg/tanka" ) @@ -43,13 +44,19 @@ func exportCmd() *cli.Command { recursive := cmd.Flags().BoolP("recursive", "r", false, "Look recursively for Tanka environments") cmd.Run = func(cmd *cli.Command, args []string) error { + filters, err := process.StrExps(vars.targets...) + if err != nil { + return err + } + opts := tanka.ExportEnvOpts{ Format: *format, Extension: *extension, Merge: *merge, - Targets: vars.targets, Opts: tanka.Opts{ JsonnetOpts: getJsonnetOpts(), + Filters: filters, + Name: vars.name, }, Selector: getLabelSelector(), Parallelism: *parallel, @@ -65,7 +72,7 @@ func exportCmd() *cli.Command { } // get absolute path to Environment - envs, err := tanka.ListEnvs(path, tanka.ListOpts{Selector: opts.Selector}) + envs, err := tanka.FindEnvs(path, tanka.FindOpts{Selector: opts.Selector}) if err != nil { return err } @@ -77,8 +84,14 @@ func exportCmd() *cli.Command { } // validate environment - if _, err := tanka.Peek(path, opts.Opts.JsonnetOpts); err != nil { - return err + if _, err := tanka.Peek(path, opts.Opts); err != nil { + switch err.(type) { + case tanka.ErrMultipleEnvs: + fmt.Println("Please use --name to export a single environment or --recursive to export multiple environments.") + return err + default: + return err + } } paths = append(paths, path) diff --git a/cmd/tk/flags.go b/cmd/tk/flags.go index aab5af384..96a005476 100644 --- a/cmd/tk/flags.go +++ b/cmd/tk/flags.go @@ -13,11 +13,13 @@ import ( ) type workflowFlagVars struct { + name string targets []string } func workflowFlags(fs *pflag.FlagSet) *workflowFlagVars { v := workflowFlagVars{} + fs.StringVar(&v.name, "name", "", "selects an environment from inline environments") fs.StringSliceVarP(&v.targets, "target", "t", nil, "only use the specified objects (Format: /)") return &v } diff --git a/cmd/tk/workflow.go b/cmd/tk/workflow.go index c66c31ab4..9d3f73cbf 100644 --- a/cmd/tk/workflow.go +++ b/cmd/tk/workflow.go @@ -43,6 +43,7 @@ func applyCmd() *cli.Command { } opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() + opts.Name = vars.name return tanka.Apply(args[0], opts) } @@ -92,6 +93,7 @@ func deleteCmd() *cli.Command { } opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() + opts.Name = vars.name return tanka.Delete(args[0], opts) } @@ -122,6 +124,7 @@ func diffCmd() *cli.Command { } opts.Filters = filters opts.JsonnetOpts = getJsonnetOpts() + opts.Name = vars.name changes, err := tanka.Diff(args[0], opts) if err != nil { @@ -173,6 +176,7 @@ Otherwise run tk show --dangerous-allow-redirect to bypass this check.`) pretty, err := tanka.Show(args[0], tanka.Opts{ JsonnetOpts: getJsonnetOpts(), Filters: filters, + Name: vars.name, }) if err != nil { diff --git a/pkg/tanka/errors.go b/pkg/tanka/errors.go index 8422b55a0..a8ae5398e 100644 --- a/pkg/tanka/errors.go +++ b/pkg/tanka/errors.go @@ -22,7 +22,7 @@ type ErrMultipleEnvs struct { } func (e ErrMultipleEnvs) Error() string { - return fmt.Sprintf("found multiple Environments (%s) in '%s'", strings.Join(e.names, ", "), e.path) + return fmt.Sprintf("found multiple Environments in '%s': \n - %s", e.path, strings.Join(e.names, "\n - ")) } // ErrParallel is an array of errors collected while parsing environments in parallel diff --git a/pkg/tanka/evaluators.go b/pkg/tanka/evaluators.go index b8f145090..e262922e1 100644 --- a/pkg/tanka/evaluators.go +++ b/pkg/tanka/evaluators.go @@ -53,8 +53,8 @@ function(%s) const PatternEvalScript = "main.%s" -// EnvsOnlyEvalScript finds the Environment object (without its .data object) -const EnvsOnlyEvalScript = ` +// MetadataEvalScript finds the Environment object (without its .data object) +const MetadataEvalScript = ` local noDataEnv(object) = std.prune( if std.isObject(object) @@ -83,3 +83,65 @@ local noDataEnv(object) = noDataEnv(main) ` + +// MetadataSingleEnvEvalScript returns a Single Environment object +const MetadataSingleEnvEvalScript = ` +local singleEnv(object) = + std.prune( + if std.isObject(object) + then + if std.objectHas(object, 'apiVersion') + && std.objectHas(object, 'kind') + then + if object.kind == 'Environment' + && object.metadata.name == '%s' + then object { data:: {} } + else {} + else + std.mapWithKey( + function(key, obj) + singleEnv(obj), + object + ) + else if std.isArray(object) + then + std.map( + function(obj) + singleEnv(obj), + object + ) + else {} + ); + +singleEnv(main) +` + +// SingleEnvEvalScript returns a Single Environment object +const SingleEnvEvalScript = ` +local singleEnv(object) = + if std.isObject(object) + then + if std.objectHas(object, 'apiVersion') + && std.objectHas(object, 'kind') + then + if object.kind == 'Environment' + && object.metadata.name == '%s' + then object + else {} + else + std.mapWithKey( + function(key, obj) + singleEnv(obj), + object + ) + else if std.isArray(object) + then + std.map( + function(obj) + singleEnv(obj), + object + ) + else {}; + +singleEnv(main) +` diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 970577a85..5466a2721 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -15,7 +15,6 @@ import ( "k8s.io/apimachinery/pkg/labels" "github.com/grafana/tanka/pkg/kubernetes/manifest" - "github.com/grafana/tanka/pkg/process" ) // BelRune is a string of the Ascii character BEL which made computers ring in ancient times @@ -36,8 +35,6 @@ type ExportEnvOpts struct { Extension string // merge export with existing directory Merge bool - // optional: only export specified Kubernetes manifests - Targets []string // optional: options to parse Jsonnet Opts Opts // optional: filter environments based on labels @@ -61,7 +58,7 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { // get all environments for paths envs, err := parallelLoadEnvironments(paths, parallelOpts{ - JsonnetOpts: opts.Opts.JsonnetOpts, + Opts: opts.Opts, Selector: opts.Selector, Parallelism: opts.Parallelism, }) @@ -70,14 +67,8 @@ func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { } for _, env := range envs { - // select targets to export - filter, err := process.StrExps(opts.Targets...) - if err != nil { - return err - } - // get the manifests - loaded, err := LoadManifests(env, filter) + loaded, err := LoadManifests(env, opts.Opts.Filters) if err != nil { return err } diff --git a/pkg/tanka/list.go b/pkg/tanka/find.go similarity index 54% rename from pkg/tanka/list.go rename to pkg/tanka/find.go index 056f3a546..36d15dbcf 100644 --- a/pkg/tanka/list.go +++ b/pkg/tanka/find.go @@ -2,25 +2,25 @@ package tanka import ( "io/ioutil" + "os" "path/filepath" - "github.com/grafana/tanka/pkg/jsonnet" "github.com/grafana/tanka/pkg/spec/v1alpha1" "k8s.io/apimachinery/pkg/labels" ) -// ListOpts are optional arguments for ListEnvs -type ListOpts struct { +// FindOpts are optional arguments for FindEnvs +type FindOpts struct { Selector labels.Selector } -// ListEnvs returns metadata of all environments recursively found in 'dir'. +// FindEnvs returns metadata of all environments recursively found in 'path'. // Each directory is tested and included if it is a valid environment, either // static or inline. If a directory is a valid environment, its subdirectories // are not checked. -func ListEnvs(dir string, opts ListOpts) ([]*v1alpha1.Environment, error) { - // list all environments at dir - envs, err := list(dir) +func FindEnvs(path string, opts FindOpts) ([]*v1alpha1.Environment, error) { + // find all environments at dir + envs, err := find(path) if err != nil { return nil, err } @@ -41,33 +41,43 @@ func ListEnvs(dir string, opts ListOpts) ([]*v1alpha1.Environment, error) { return filtered, nil } -// list implements the actual functionality described at 'ListEnvs' -func list(dir string) ([]*v1alpha1.Environment, error) { - // list directory, also checks if dir - files, err := ioutil.ReadDir(dir) +// find implements the actual functionality described at 'FindEnvs' +func find(path string) ([]*v1alpha1.Environment, error) { + // try if this has envs + list, err := List(path, Opts{}) + if len(list) != 0 && err == nil { + // it has. don't search deeper + return list, nil + } + + stat, err := os.Stat(path) if err != nil { return nil, err } - // try if this is an env - env, err := Peek(dir, jsonnet.Opts{}) - if err == nil { - // it is one. don't search deeper - return []*v1alpha1.Environment{env}, nil + // if path is a file, don't search deeper + if !stat.IsDir() { + return nil, nil + } + + // list directory + files, err := ioutil.ReadDir(path) + if err != nil { + return nil, err } // it's not one. Maybe subdirectories are? - ch := make(chan listOut) + ch := make(chan findOut) routines := 0 - // recursively list in parallel + // recursively find in parallel for _, fi := range files { if !fi.IsDir() { continue } routines++ - go listShim(filepath.Join(dir, fi.Name()), ch) + go findShim(filepath.Join(path, fi.Name()), ch) } // collect parallel results @@ -90,12 +100,12 @@ func list(dir string) ([]*v1alpha1.Environment, error) { return envs, nil } -type listOut struct { +type findOut struct { envs []*v1alpha1.Environment err error } -func listShim(dir string, ch chan listOut) { - envs, err := list(dir) - ch <- listOut{envs: envs, err: err} +func findShim(dir string, ch chan findOut) { + envs, err := find(dir) + ch <- findOut{envs: envs, err: err} } diff --git a/pkg/tanka/inline.go b/pkg/tanka/inline.go index 4137b4c6a..871e8e8e4 100644 --- a/pkg/tanka/inline.go +++ b/pkg/tanka/inline.go @@ -17,18 +17,12 @@ import ( // Kubernetes resources are expected at the `data` key of this very type type InlineLoader struct{} -func (i *InlineLoader) Load(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { - raw, err := EvalJsonnet(path, opts) - if err != nil { - return nil, err - } - - var data interface{} - if err := json.Unmarshal([]byte(raw), &data); err != nil { - return nil, err +func (i *InlineLoader) Load(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { + if opts.Name != "" { + opts.JsonnetOpts.EvalScript = fmt.Sprintf(SingleEnvEvalScript, opts.Name) } - envs, err := extractEnvs(data) + envs, err := inlineEval(path, opts.JsonnetOpts) if err != nil { return nil, err } @@ -45,38 +39,94 @@ func (i *InlineLoader) Load(path string, opts JsonnetOpts) (*v1alpha1.Environmen return nil, fmt.Errorf("Found no environments in '%s'", path) } - root, err := jpath.FindRoot(path) + // TODO: Re-serializing the entire env here. This is horribly inefficient + envData, err := json.Marshal(envs[0]) if err != nil { return nil, err } - file, err := jpath.Entrypoint(path) + env, err := inlineParse(path, envData) if err != nil { return nil, err } - namespace, err := filepath.Rel(root, file) + return env, nil +} + +func (i *InlineLoader) Peek(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { + opts.JsonnetOpts.EvalScript = MetadataEvalScript + if opts.Name != "" { + opts.JsonnetOpts.EvalScript = fmt.Sprintf(MetadataSingleEnvEvalScript, opts.Name) + } + return i.Load(path, opts) +} + +func (i *InlineLoader) List(path string, opts LoaderOpts) ([]*v1alpha1.Environment, error) { + opts.JsonnetOpts.EvalScript = MetadataEvalScript + list, err := inlineEval(path, opts.JsonnetOpts) if err != nil { return nil, err } - // TODO: Re-serializing the entire env here. This is horribly inefficient - envData, err := json.Marshal(envs[0]) + envs := make([]*v1alpha1.Environment, 0, len(list)) + for _, raw := range list { + data, err := json.Marshal(raw) + if err != nil { + return nil, err + } + + env, err := inlineParse(path, data) + if err != nil { + return nil, err + } + + envs = append(envs, env) + } + + return envs, nil +} + +func inlineEval(path string, opts JsonnetOpts) (manifest.List, error) { + raw, err := EvalJsonnet(path, opts) if err != nil { return nil, err } - env, err := spec.Parse(envData, namespace) + var data interface{} + if err := json.Unmarshal([]byte(raw), &data); err != nil { + return nil, err + } + + envs, err := extractEnvs(data) if err != nil { return nil, err } - return env, nil + return envs, nil } -func (i *InlineLoader) Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { - opts.EvalScript = EnvsOnlyEvalScript - return i.Load(path, opts) +func inlineParse(path string, data []byte) (*v1alpha1.Environment, error) { + root, err := jpath.FindRoot(path) + if err != nil { + return nil, err + } + + file, err := jpath.Entrypoint(path) + if err != nil { + return nil, err + } + + namespace, err := filepath.Rel(root, file) + if err != nil { + return nil, err + } + + env, err := spec.Parse(data, namespace) + if err != nil { + return nil, err + } + + return env, nil } // extractEnvs filters out any Environment manifests diff --git a/pkg/tanka/load.go b/pkg/tanka/load.go index 3a37598cc..fe0f55a45 100644 --- a/pkg/tanka/load.go +++ b/pkg/tanka/load.go @@ -36,7 +36,7 @@ func LoadEnvironment(path string, opts Opts) (*v1alpha1.Environment, error) { return nil, err } - env, err := loader.Load(path, opts.JsonnetOpts) + env, err := loader.Load(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) if err != nil { return nil, err } @@ -59,13 +59,25 @@ func LoadManifests(env *v1alpha1.Environment, filters process.Matchers) (*LoadRe // Peek loads the metadata of the environment at path. To get resources as well, // use Load -func Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { +func Peek(path string, opts Opts) (*v1alpha1.Environment, error) { loader, err := DetectLoader(path) if err != nil { return nil, err } - return loader.Peek(path, opts) + return loader.Peek(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) +} + +// List finds metadata of all environments at path that could possibly be +// loaded. List can be used to deal with multiple inline environments, by first +// listing them, choosing the right one and then only loading that one +func List(path string, opts Opts) ([]*v1alpha1.Environment, error) { + loader, err := DetectLoader(path) + if err != nil { + return nil, err + } + + return loader.List(path, LoaderOpts{opts.JsonnetOpts, opts.Name}) } // DetectLoader detects whether the environment is inline or static and picks @@ -89,11 +101,20 @@ func DetectLoader(path string) (Loader, error) { // Loader is an abstraction over the process of loading Environments type Loader interface { - // Load the environment with path - Load(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) + // Load a single environment at path + Load(path string, opts LoaderOpts) (*v1alpha1.Environment, error) // Peek only loads metadata and omits the actual resources - Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) + Peek(path string, opts LoaderOpts) (*v1alpha1.Environment, error) + + // List returns metadata of all possible environments at path that can be + // loaded + List(path string, opts LoaderOpts) ([]*v1alpha1.Environment, error) +} + +type LoaderOpts struct { + JsonnetOpts + Name string } type LoadResult struct { diff --git a/pkg/tanka/parallel.go b/pkg/tanka/parallel.go index 0ba6e78c9..3ddbd6953 100644 --- a/pkg/tanka/parallel.go +++ b/pkg/tanka/parallel.go @@ -11,7 +11,7 @@ import ( const defaultParallelism = 8 type parallelOpts struct { - JsonnetOpts JsonnetOpts + Opts Selector labels.Selector Parallelism int } @@ -19,7 +19,20 @@ type parallelOpts struct { // parallelLoadEnvironments evaluates multiple environments in parallel func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.Environment, error) { jobsCh := make(chan parallelJob) - outCh := make(chan parallelOut, len(paths)) + list := make(map[string]string) + for _, path := range paths { + envs, err := FindEnvs(path, FindOpts{opts.Selector}) + if err != nil { + return nil, err + } + for _, env := range envs { + if opts.Name != "" && opts.Name != env.Metadata.Name { + continue + } + list[env.Metadata.Name] = path + } + } + outCh := make(chan parallelOut, len(list)) if opts.Parallelism <= 0 { opts.Parallelism = defaultParallelism @@ -29,17 +42,19 @@ func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.En go parallelWorker(jobsCh, outCh) } - for _, path := range paths { + for name, path := range list { + o := opts.Opts + o.Name = name jobsCh <- parallelJob{ path: path, - opts: Opts{JsonnetOpts: opts.JsonnetOpts}, + opts: o, } } close(jobsCh) var envs []*v1alpha1.Environment var errors []error - for i := 0; i < len(paths); i++ { + for i := 0; i < len(list); i++ { out := <-outCh if out.err != nil { errors = append(errors, out.err) diff --git a/pkg/tanka/static.go b/pkg/tanka/static.go index 445ca0f98..cadb2550c 100644 --- a/pkg/tanka/static.go +++ b/pkg/tanka/static.go @@ -12,13 +12,13 @@ import ( // Jsonnet is evaluated as normal type StaticLoader struct{} -func (s StaticLoader) Load(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { - config, err := Peek(path, opts) +func (s StaticLoader) Load(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { + config, err := s.Peek(path, opts) if err != nil { return nil, err } - data, err := EvalJsonnet(path, opts) + data, err := EvalJsonnet(path, opts.JsonnetOpts) if err != nil { return nil, err } @@ -30,7 +30,7 @@ func (s StaticLoader) Load(path string, opts JsonnetOpts) (*v1alpha1.Environment return config, nil } -func (s StaticLoader) Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment, error) { +func (s StaticLoader) Peek(path string, opts LoaderOpts) (*v1alpha1.Environment, error) { config, err := parseStaticSpec(path) if err != nil { return nil, err @@ -39,6 +39,15 @@ func (s StaticLoader) Peek(path string, opts JsonnetOpts) (*v1alpha1.Environment return config, nil } +func (s StaticLoader) List(path string, opts LoaderOpts) ([]*v1alpha1.Environment, error) { + env, err := s.Peek(path, opts) + if err != nil { + return nil, err + } + + return []*v1alpha1.Environment{env}, nil +} + // parseStaticSpec parses the `spec.json` of the environment and returns a // *kubernetes.Kubernetes from it func parseStaticSpec(path string) (*v1alpha1.Environment, error) { diff --git a/pkg/tanka/tanka.go b/pkg/tanka/tanka.go index 36ce69753..d7f91a088 100644 --- a/pkg/tanka/tanka.go +++ b/pkg/tanka/tanka.go @@ -21,6 +21,9 @@ type Opts struct { // Filters are used to optionally select a subset of the resources Filters process.Matchers + + // Name is used to extract a single environment from multiple environments + Name string } // DEFAULT_DEV_VERSION is the placeholder version used when no actual semver is