Enable multi config support in Skaffold (GoogleContainerTools#5160)

* wip

* fix tests

* revert example change

* fix for non-managed images

* fix `InitMeterFromConfig`

* fix message in `fix` command

* address PR feedback

* add UTs

* rename `Pipeline` to `PipelineForImage`

cmd/skaffold/app/cmd/build.go

@@ -65,8 +65,8 @@ func doBuild(ctx context.Context, out io.Writer) error {
 		buildOut = ioutil.Discard
 	}
 
-	return withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error {
-		bRes, err := r.Build(ctx, buildOut, targetArtifacts(opts, config))
+	return withRunner(ctx, func(r runner.Runner, configs []*latest.SkaffoldConfig) error {
+		bRes, err := r.Build(ctx, buildOut, targetArtifacts(opts, configs))
 
 		if quietFlag || buildOutputFlag != "" {
 			cmdOut := flags.BuildOutput{Builds: bRes}
@@ -92,14 +92,14 @@ func doBuild(ctx context.Context, out io.Writer) error {
 	})
 }
 
-func targetArtifacts(opts config.SkaffoldOptions, cfg *latest.SkaffoldConfig) []*latest.Artifact {
+func targetArtifacts(opts config.SkaffoldOptions, configs []*latest.SkaffoldConfig) []*latest.Artifact {
 	var targetArtifacts []*latest.Artifact
-
-	for _, artifact := range cfg.Build.Artifacts {
-		if opts.IsTargetImage(artifact) {
-			targetArtifacts = append(targetArtifacts, artifact)
+	for _, cfg := range configs {
+		for _, artifact := range cfg.Build.Artifacts {
+			if opts.IsTargetImage(artifact) {
+				targetArtifacts = append(targetArtifacts, artifact)
+			}
 		}
 	}
-
 	return targetArtifacts
 }

cmd/skaffold/app/cmd/build_test.go

@@ -49,8 +49,8 @@ func (r *mockRunner) Stop() error {
 }
 
 func TestTagFlag(t *testing.T) {
-	mockCreateRunner := func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
-		return &mockRunner{}, &latest.SkaffoldConfig{}, nil
+	mockCreateRunner := func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
+		return &mockRunner{}, []*latest.SkaffoldConfig{{}}, nil
 	}
 
 	testutil.Run(t, "override tag with argument", func(t *testutil.T) {
@@ -68,8 +68,8 @@ func TestTagFlag(t *testing.T) {
 }
 
 func TestQuietFlag(t *testing.T) {
-	mockCreateRunner := func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
-		return &mockRunner{}, &latest.SkaffoldConfig{}, nil
+	mockCreateRunner := func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
+		return &mockRunner{}, []*latest.SkaffoldConfig{{}}, nil
 	}
 
 	tests := []struct {
@@ -114,8 +114,8 @@ func TestQuietFlag(t *testing.T) {
 }
 
 func TestFileOutputFlag(t *testing.T) {
-	mockCreateRunner := func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
-		return &mockRunner{}, &latest.SkaffoldConfig{}, nil
+	mockCreateRunner := func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
+		return &mockRunner{}, []*latest.SkaffoldConfig{{}}, nil
 	}
 
 	tests := []struct {
@@ -177,16 +177,16 @@ func TestFileOutputFlag(t *testing.T) {
 }
 
 func TestRunBuild(t *testing.T) {
-	errRunner := func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
+	errRunner := func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
 		return nil, nil, errors.New("some error")
 	}
-	mockCreateRunner := func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
-		return &mockRunner{}, &latest.SkaffoldConfig{}, nil
+	mockCreateRunner := func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
+		return &mockRunner{}, []*latest.SkaffoldConfig{{}}, nil
 	}
 
 	tests := []struct {
 		description string
-		mock        func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error)
+		mock        func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error)
 		shouldErr   bool
 	}{
 		{

cmd/skaffold/app/cmd/debug_test.go

@@ -48,8 +48,8 @@ func TestNewCmdDebug(t *testing.T) {
 func TestDebugIndependentFromDev(t *testing.T) {
 	mockRunner := &mockDevRunner{}
 	testutil.Run(t, "DevDebug", func(t *testutil.T) {
-		t.Override(&createRunner, func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
-			return mockRunner, &latest.SkaffoldConfig{}, nil
+		t.Override(&createRunner, func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
+			return mockRunner, []*latest.SkaffoldConfig{{}}, nil
 		})
 		t.Override(&opts, config.SkaffoldOptions{})
 		t.Override(&doDev, func(context.Context, io.Writer) error {

cmd/skaffold/app/cmd/delete.go

@@ -35,7 +35,7 @@ func NewCmdDelete() *cobra.Command {
 }
 
 func doDelete(ctx context.Context, out io.Writer) error {
-	return withRunner(ctx, func(r runner.Runner, _ *latest.SkaffoldConfig) error {
+	return withRunner(ctx, func(r runner.Runner, _ []*latest.SkaffoldConfig) error {
 		return r.Cleanup(ctx, out)
 	})
 }

cmd/skaffold/app/cmd/deploy.go

@@ -51,12 +51,15 @@ func NewCmdDeploy() *cobra.Command {
 }
 
 func doDeploy(ctx context.Context, out io.Writer) error {
-	return withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error {
+	return withRunner(ctx, func(r runner.Runner, configs []*latest.SkaffoldConfig) error {
 		if opts.SkipRender {
 			return r.DeployAndLog(ctx, out, []build.Artifact{})
 		}
-
-		buildArtifacts, err := getBuildArtifactsAndSetTags(r, config)
+		var artifacts []*latest.Artifact
+		for _, cfg := range configs {
+			artifacts = append(artifacts, cfg.Build.Artifacts...)
+		}
+		buildArtifacts, err := getBuildArtifactsAndSetTags(r, artifacts)
 		if err != nil {
 			tips.PrintUseRunVsDeploy(out)
 			return err

cmd/skaffold/app/cmd/dev.go

@@ -60,8 +60,12 @@ func runDev(ctx context.Context, out io.Writer) error {
 		case <-ctx.Done():
 			return nil
 		default:
-			err := withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error {
-				err := r.Dev(ctx, out, config.Build.Artifacts)
+			err := withRunner(ctx, func(r runner.Runner, configs []*latest.SkaffoldConfig) error {
+				var artifacts []*latest.Artifact
+				for _, cfg := range configs {
+					artifacts = append(artifacts, cfg.Build.Artifacts...)
+				}
+				err := r.Dev(ctx, out, artifacts)
 
 				if r.HasDeployed() {
 					cleanup = func() {

cmd/skaffold/app/cmd/dev_test.go

@@ -95,8 +95,8 @@ func TestDoDev(t *testing.T) {
 				hasDeployed: test.hasDeployed,
 				errDev:      context.Canceled,
 			}
-			t.Override(&createRunner, func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
-				return mockRunner, &latest.SkaffoldConfig{}, nil
+			t.Override(&createRunner, func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
+				return mockRunner, []*latest.SkaffoldConfig{{}}, nil
 			})
 			t.Override(&opts, config.SkaffoldOptions{
 				Cleanup: true,
@@ -145,8 +145,8 @@ func TestDevConfigChange(t *testing.T) {
 	testutil.Run(t, "test config change", func(t *testutil.T) {
 		mockRunner := &mockConfigChangeRunner{}
 
-		t.Override(&createRunner, func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
-			return mockRunner, &latest.SkaffoldConfig{}, nil
+		t.Override(&createRunner, func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
+			return mockRunner, []*latest.SkaffoldConfig{{}}, nil
 		})
 		t.Override(&opts, config.SkaffoldOptions{
 			Cleanup: true,

cmd/skaffold/app/cmd/diagnose.go

@@ -46,24 +46,25 @@ func NewCmdDiagnose() *cobra.Command {
 }
 
 func doDiagnose(ctx context.Context, out io.Writer) error {
-	runCtx, config, err := runContext(opts)
+	runCtx, configs, err := runContext(opts)
 	if err != nil {
 		return err
 	}
 
-	if !yamlOnly {
-		fmt.Fprintln(out, "Skaffold version:", version.Get().GitCommit)
-		fmt.Fprintln(out, "Configuration version:", config.APIVersion)
-		fmt.Fprintln(out, "Number of artifacts:", len(config.Build.Artifacts))
+	for _, config := range configs {
+		if !yamlOnly {
+			fmt.Fprintln(out, "Skaffold version:", version.Get().GitCommit)
+			fmt.Fprintln(out, "Configuration version:", config.APIVersion)
+			fmt.Fprintln(out, "Number of artifacts:", len(config.Build.Artifacts))
 
-		if err := diagnose.CheckArtifacts(ctx, runCtx, out); err != nil {
-			return fmt.Errorf("running diagnostic on artifacts: %w", err)
-		}
+			if err := diagnose.CheckArtifacts(ctx, runCtx, out); err != nil {
+				return fmt.Errorf("running diagnostic on artifacts: %w", err)
+			}
 
-		color.Blue.Fprintln(out, "\nConfiguration")
+			color.Blue.Fprintln(out, "\nConfiguration")
+		}
 	}
-
-	buf, err := yaml.Marshal(config)
+	buf, err := yaml.MarshalWithSeparator(configs)
 	if err != nil {
 		return fmt.Errorf("marshalling configuration: %w", err)
 	}

cmd/skaffold/app/cmd/filter.go

@@ -58,7 +58,7 @@ func NewCmdFilter() *cobra.Command {
 // runFilter loads the Kubernetes manifests from stdin and applies the debug transformations.
 // Unlike `skaffold debug`, this filtering affects all images and not just the built artifacts.
 func runFilter(ctx context.Context, out io.Writer, debuggingFilters bool, buildArtifacts []build.Artifact) error {
-	return withRunner(ctx, func(r runner.Runner, cfg *latest.SkaffoldConfig) error {
+	return withRunner(ctx, func(r runner.Runner, configs []*latest.SkaffoldConfig) error {
 		manifestList, err := manifest.Load(os.Stdin)
 		if err != nil {
 			return fmt.Errorf("loading manifests: %w", err)
@@ -69,7 +69,7 @@ func runFilter(ctx context.Context, out io.Writer, debuggingFilters bool, buildA
 			if err != nil {
 				return fmt.Errorf("resolving debug helpers: %w", err)
 			}
-			insecureRegistries, err := getInsecureRegistries(opts, cfg)
+			insecureRegistries, err := getInsecureRegistries(opts, configs)
 			if err != nil {
 				return fmt.Errorf("retrieving insecure registries: %w", err)
 			}
@@ -87,12 +87,17 @@ func runFilter(ctx context.Context, out io.Writer, debuggingFilters bool, buildA
 	})
 }
 
-func getInsecureRegistries(opts config.SkaffoldOptions, cfg *latest.SkaffoldConfig) (map[string]bool, error) {
+func getInsecureRegistries(opts config.SkaffoldOptions, configs []*latest.SkaffoldConfig) (map[string]bool, error) {
 	cfgRegistries, err := config.GetInsecureRegistries(opts.GlobalConfig)
 	if err != nil {
 		return nil, err
 	}
-	regList := append(opts.InsecureRegistries, cfg.Build.InsecureRegistries...)
+	var regList []string
+
+	regList = append(regList, opts.InsecureRegistries...)
+	for _, cfg := range configs {
+		regList = append(regList, cfg.Build.InsecureRegistries...)
+	}
 	regList = append(regList, cfgRegistries...)
 	insecureRegistries := make(map[string]bool, len(regList))
 	for _, r := range regList {

cmd/skaffold/app/cmd/find_configs.go

@@ -88,8 +88,9 @@ func findConfigs(directory string) (map[string]string, error) {
 	}
 
 	err := walk.From(directory).When(isYaml).Do(func(path string, _ walk.Dirent) error {
-		if cfg, err := schema.ParseConfig(path); err == nil {
-			pathToVersion[path] = cfg.GetVersion()
+		if cfgs, err := schema.ParseConfig(path); err == nil && len(cfgs) > 0 {
+			// all configs defined in the same file should have the same version
+			pathToVersion[path] = cfgs[0].GetVersion()
 		}
 		return nil
 	})

cmd/skaffold/app/cmd/fix.go

@@ -51,39 +51,47 @@ func doFix(_ context.Context, out io.Writer) error {
 }
 
 func fix(out io.Writer, configFile string, toVersion string, overwrite bool) error {
-	cfg, err := schema.ParseConfig(configFile)
+	parsedCfgs, err := schema.ParseConfig(configFile)
 	if err != nil {
 		return err
 	}
-
-	if cfg.GetVersion() == toVersion {
+	needsUpdate := false
+	for _, cfg := range parsedCfgs {
+		if cfg.GetVersion() != toVersion {
+			needsUpdate = true
+			break
+		}
+	}
+	if !needsUpdate {
 		color.Default.Fprintln(out, "config is already version", toVersion)
 		return nil
 	}
 
-	cfg, err = schema.ParseConfigAndUpgrade(configFile, toVersion)
+	versionedCfgs, err := schema.ParseConfigAndUpgrade(configFile, toVersion)
 	if err != nil {
 		return err
 	}
 
 	// TODO(dgageot): We should be able run validations on any schema version
 	// but that's not the case. They can only run on the latest version for now.
 	if toVersion == latest.Version {
-		if err := validation.Process(cfg.(*latest.SkaffoldConfig)); err != nil {
+		var cfgs []*latest.SkaffoldConfig
+		for _, cfg := range versionedCfgs {
+			cfgs = append(cfgs, cfg.(*latest.SkaffoldConfig))
+		}
+		if err := validation.Process(cfgs); err != nil {
 			return fmt.Errorf("validating upgraded config: %w", err)
 		}
 	}
-
-	newCfg, err := yaml.Marshal(cfg)
+	newCfg, err := yaml.MarshalWithSeparator(versionedCfgs)
 	if err != nil {
 		return fmt.Errorf("marshaling new config: %w", err)
 	}
-
 	if overwrite {
 		if err := ioutil.WriteFile(configFile, newCfg, 0644); err != nil {
 			return fmt.Errorf("writing config file: %w", err)
 		}
-		color.Default.Fprintf(out, "New config at version %s generated and written to %s\n", cfg.GetVersion(), opts.ConfigurationFile)
+		color.Default.Fprintf(out, "New config at version %s generated and written to %s\n", toVersion, opts.ConfigurationFile)
 	} else {
 		out.Write(newCfg)
 	}

cmd/skaffold/app/cmd/generate_pipeline.go

@@ -44,8 +44,8 @@ func NewCmdGeneratePipeline() *cobra.Command {
 }
 
 func doGeneratePipeline(ctx context.Context, out io.Writer) error {
-	return withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error {
-		if err := r.GeneratePipeline(ctx, out, config, configFiles, "pipeline.yaml"); err != nil {
+	return withRunner(ctx, func(r runner.Runner, configs []*latest.SkaffoldConfig) error {
+		if err := r.GeneratePipeline(ctx, out, configs, configFiles, "pipeline.yaml"); err != nil {
 			return fmt.Errorf("generating : %w", err)
 		}
 		color.Default.Fprintln(out, "Pipeline config written to pipeline.yaml!")

cmd/skaffold/app/cmd/render.go

@@ -60,14 +60,14 @@ func doRender(ctx context.Context, out io.Writer) error {
 		buildOut = out
 	}
 
-	return withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error {
+	return withRunner(ctx, func(r runner.Runner, configs []*latest.SkaffoldConfig) error {
 		var bRes []build.Artifact
 
 		if renderFromBuildOutputFile.String() != "" {
 			bRes = renderFromBuildOutputFile.BuildArtifacts()
 		} else {
 			var err error
-			bRes, err = r.Build(ctx, buildOut, targetArtifacts(opts, config))
+			bRes, err = r.Build(ctx, buildOut, targetArtifacts(opts, configs))
 			if err != nil {
 				return fmt.Errorf("executing build: %w", err)
 			}

cmd/skaffold/app/cmd/run.go

@@ -41,8 +41,8 @@ func NewCmdRun() *cobra.Command {
 }
 
 func doRun(ctx context.Context, out io.Writer) error {
-	return withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error {
-		bRes, err := r.Build(ctx, out, targetArtifacts(opts, config))
+	return withRunner(ctx, func(r runner.Runner, configs []*latest.SkaffoldConfig) error {
+		bRes, err := r.Build(ctx, out, targetArtifacts(opts, configs))
 		if err != nil {
 			return fmt.Errorf("failed to build: %w", err)
 		}

cmd/skaffold/app/cmd/run_test.go

@@ -69,8 +69,8 @@ func (r *mockRunRunner) DeployAndLog(context.Context, io.Writer, []build.Artifac
 func TestBuildImageFlag(t *testing.T) {
 	testutil.Run(t, "", func(t *testutil.T) {
 		mockRunner := &mockRunRunner{}
-		t.Override(&createRunner, func(config.SkaffoldOptions) (runner.Runner, *latest.SkaffoldConfig, error) {
-			return mockRunner, &latest.SkaffoldConfig{
+		t.Override(&createRunner, func(config.SkaffoldOptions) (runner.Runner, []*latest.SkaffoldConfig, error) {
+			return mockRunner, []*latest.SkaffoldConfig{{
 				Pipeline: latest.Pipeline{
 					Build: latest.BuildConfig{
 						Artifacts: []*latest.Artifact{
@@ -81,7 +81,7 @@ func TestBuildImageFlag(t *testing.T) {
 						},
 					},
 				},
-			}, nil
+			}}, nil
 		})
 		t.Override(&opts, config.SkaffoldOptions{
 			TargetImages: []string{"test"},