From 4d7e58e144e9c3537f0f57bd3c7c12383d978e4a Mon Sep 17 00:00:00 2001 From: Duologic Date: Mon, 12 Apr 2021 11:15:37 +0200 Subject: [PATCH 1/2] perf(export): only call FindEnvs once --- cmd/tk/export.go | 22 +++++++++++++--------- pkg/tanka/export.go | 2 +- pkg/tanka/parallel.go | 23 ++++++----------------- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 0560074b2..0e6f8dc97 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -62,15 +62,15 @@ func exportCmd() *cli.Command { Parallelism: *parallel, } - var paths []string + paths := make(map[string]string) for _, path := range args[1:] { + rootDir, err := jpath.FindRoot(path) + if err != nil { + return errors.Wrap(err, "resolving jpath") + } + // find possible environments if *recursive { - rootDir, err := jpath.FindRoot(path) - if err != nil { - return errors.Wrap(err, "resolving jpath") - } - // get absolute path to Environment envs, err := tanka.FindEnvs(path, tanka.FindOpts{Selector: opts.Selector}) if err != nil { @@ -78,13 +78,17 @@ func exportCmd() *cli.Command { } for _, env := range envs { - paths = append(paths, filepath.Join(rootDir, env.Metadata.Namespace)) + if opts.Opts.Name != "" && opts.Opts.Name != env.Metadata.Name { + continue + } + paths[env.Metadata.Name] = filepath.Join(rootDir, env.Metadata.Namespace) } continue } // validate environment - if _, err := tanka.Peek(path, opts.Opts); err != nil { + env, err := tanka.Peek(path, opts.Opts) + if err != nil { switch err.(type) { case tanka.ErrMultipleEnvs: fmt.Println("Please use --name to export a single environment or --recursive to export multiple environments.") @@ -94,7 +98,7 @@ func exportCmd() *cli.Command { } } - paths = append(paths, path) + paths[env.Metadata.Name] = filepath.Join(rootDir, env.Metadata.Namespace) } // export them diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 5466a2721..08f186421 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -43,7 +43,7 @@ type ExportEnvOpts struct { Parallelism int } -func ExportEnvironments(paths []string, to string, opts *ExportEnvOpts) error { +func ExportEnvironments(paths map[string]string, to string, opts *ExportEnvOpts) error { // Keep track of which file maps to which environment fileToEnv := map[string]string{} diff --git a/pkg/tanka/parallel.go b/pkg/tanka/parallel.go index 2d23f5ea1..69854a518 100644 --- a/pkg/tanka/parallel.go +++ b/pkg/tanka/parallel.go @@ -2,6 +2,7 @@ package tanka import ( "fmt" + "log" "k8s.io/apimachinery/pkg/labels" @@ -17,22 +18,9 @@ type parallelOpts struct { } // parallelLoadEnvironments evaluates multiple environments in parallel -func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.Environment, error) { +func parallelLoadEnvironments(paths map[string]string, opts parallelOpts) ([]*v1alpha1.Environment, error) { jobsCh := make(chan parallelJob) - list := make(map[string]string) - for _, path := range paths { - envs, err := FindEnvs(path, FindOpts{Selector: opts.Selector, JsonnetOpts: opts.JsonnetOpts}) - if err != nil { - return nil, err - } - for _, env := range envs { - if opts.Name != "" && opts.Name != env.Metadata.Name { - continue - } - list[env.Metadata.Name] = path - } - } - outCh := make(chan parallelOut, len(list)) + outCh := make(chan parallelOut, len(paths)) if opts.Parallelism <= 0 { opts.Parallelism = defaultParallelism @@ -42,7 +30,7 @@ func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.En go parallelWorker(jobsCh, outCh) } - for name, path := range list { + for name, path := range paths { o := opts.Opts // TODO: This is required because the map[string]string in here is not @@ -62,7 +50,7 @@ func parallelLoadEnvironments(paths []string, opts parallelOpts) ([]*v1alpha1.En var envs []*v1alpha1.Environment var errors []error - for i := 0; i < len(list); i++ { + for i := 0; i < len(paths); i++ { out := <-outCh if out.err != nil { errors = append(errors, out.err) @@ -92,6 +80,7 @@ type parallelOut struct { func parallelWorker(jobsCh <-chan parallelJob, outCh chan parallelOut) { for job := range jobsCh { + log.Printf("Loading %s from %s", job.opts.Name, job.path) env, err := LoadEnvironment(job.path, job.opts) if err != nil { err = fmt.Errorf("%s:\n %w", job.path, err) From 4d360fce0ab3a873729ab68e2c3c5c378ff47b18 Mon Sep 17 00:00:00 2001 From: Duologic Date: Mon, 12 Apr 2021 13:29:15 +0200 Subject: [PATCH 2/2] envs instead of paths --- cmd/tk/export.go | 17 +++++------------ pkg/tanka/export.go | 7 ++++--- pkg/tanka/parallel.go | 28 ++++++++++++++++++---------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/cmd/tk/export.go b/cmd/tk/export.go index 0e6f8dc97..f13048e90 100644 --- a/cmd/tk/export.go +++ b/cmd/tk/export.go @@ -2,13 +2,11 @@ package main import ( "fmt" - "path/filepath" "github.com/go-clix/cli" - "github.com/pkg/errors" - "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/process" + "github.com/grafana/tanka/pkg/spec/v1alpha1" "github.com/grafana/tanka/pkg/tanka" ) @@ -62,13 +60,8 @@ func exportCmd() *cli.Command { Parallelism: *parallel, } - paths := make(map[string]string) + var exportEnvs []*v1alpha1.Environment for _, path := range args[1:] { - rootDir, err := jpath.FindRoot(path) - if err != nil { - return errors.Wrap(err, "resolving jpath") - } - // find possible environments if *recursive { // get absolute path to Environment @@ -81,7 +74,7 @@ func exportCmd() *cli.Command { if opts.Opts.Name != "" && opts.Opts.Name != env.Metadata.Name { continue } - paths[env.Metadata.Name] = filepath.Join(rootDir, env.Metadata.Namespace) + exportEnvs = append(exportEnvs, env) } continue } @@ -98,11 +91,11 @@ func exportCmd() *cli.Command { } } - paths[env.Metadata.Name] = filepath.Join(rootDir, env.Metadata.Namespace) + exportEnvs = append(exportEnvs, env) } // export them - return tanka.ExportEnvironments(paths, args[0], &opts) + return tanka.ExportEnvironments(exportEnvs, args[0], &opts) } return cmd } diff --git a/pkg/tanka/export.go b/pkg/tanka/export.go index 08f186421..8916bd0ac 100644 --- a/pkg/tanka/export.go +++ b/pkg/tanka/export.go @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "github.com/grafana/tanka/pkg/kubernetes/manifest" + "github.com/grafana/tanka/pkg/spec/v1alpha1" ) // BelRune is a string of the Ascii character BEL which made computers ring in ancient times @@ -43,7 +44,7 @@ type ExportEnvOpts struct { Parallelism int } -func ExportEnvironments(paths map[string]string, to string, opts *ExportEnvOpts) error { +func ExportEnvironments(envs []*v1alpha1.Environment, to string, opts *ExportEnvOpts) error { // Keep track of which file maps to which environment fileToEnv := map[string]string{} @@ -57,7 +58,7 @@ func ExportEnvironments(paths map[string]string, to string, opts *ExportEnvOpts) } // get all environments for paths - envs, err := parallelLoadEnvironments(paths, parallelOpts{ + loadedEnvs, err := parallelLoadEnvironments(envs, parallelOpts{ Opts: opts.Opts, Selector: opts.Selector, Parallelism: opts.Parallelism, @@ -66,7 +67,7 @@ func ExportEnvironments(paths map[string]string, to string, opts *ExportEnvOpts) return err } - for _, env := range envs { + for _, env := range loadedEnvs { // get the manifests loaded, err := LoadManifests(env, opts.Opts.Filters) if err != nil { diff --git a/pkg/tanka/parallel.go b/pkg/tanka/parallel.go index 69854a518..4fb0a1cc7 100644 --- a/pkg/tanka/parallel.go +++ b/pkg/tanka/parallel.go @@ -3,10 +3,13 @@ package tanka import ( "fmt" "log" + "path/filepath" "k8s.io/apimachinery/pkg/labels" + "github.com/grafana/tanka/pkg/jsonnet/jpath" "github.com/grafana/tanka/pkg/spec/v1alpha1" + "github.com/pkg/errors" ) const defaultParallelism = 8 @@ -18,9 +21,9 @@ type parallelOpts struct { } // parallelLoadEnvironments evaluates multiple environments in parallel -func parallelLoadEnvironments(paths map[string]string, opts parallelOpts) ([]*v1alpha1.Environment, error) { +func parallelLoadEnvironments(envs []*v1alpha1.Environment, opts parallelOpts) ([]*v1alpha1.Environment, error) { jobsCh := make(chan parallelJob) - outCh := make(chan parallelOut, len(paths)) + outCh := make(chan parallelOut, len(envs)) if opts.Parallelism <= 0 { opts.Parallelism = defaultParallelism @@ -30,7 +33,7 @@ func parallelLoadEnvironments(paths map[string]string, opts parallelOpts) ([]*v1 go parallelWorker(jobsCh, outCh) } - for name, path := range paths { + for _, env := range envs { o := opts.Opts // TODO: This is required because the map[string]string in here is not @@ -40,32 +43,37 @@ func parallelLoadEnvironments(paths map[string]string, opts parallelOpts) ([]*v1 // to Tanka workflow thus being able to handle such cases o.JsonnetOpts = o.JsonnetOpts.Clone() - o.Name = name + o.Name = env.Metadata.Name + path := env.Metadata.Namespace + rootDir, err := jpath.FindRoot(path) + if err != nil { + return nil, errors.Wrap(err, "finding root") + } jobsCh <- parallelJob{ - path: path, + path: filepath.Join(rootDir, path), opts: o, } } close(jobsCh) - var envs []*v1alpha1.Environment + var outenvs []*v1alpha1.Environment var errors []error - for i := 0; i < len(paths); i++ { + for i := 0; i < len(envs); i++ { out := <-outCh if out.err != nil { errors = append(errors, out.err) continue } if opts.Selector == nil || opts.Selector.Empty() || opts.Selector.Matches(out.env.Metadata) { - envs = append(envs, out.env) + outenvs = append(outenvs, out.env) } } if len(errors) != 0 { - return envs, ErrParallel{errors: errors} + return outenvs, ErrParallel{errors: errors} } - return envs, nil + return outenvs, nil } type parallelJob struct {