Substituting variables into the devfile from the CLI (#5749)

* Define var and and-file flags

* vars package

* Vars for deploy

* Add integration tests for dev

* Update dev mock

* New devfile library version

* Add doc

* Fix validate

* Review

* Apply suggestions from code review

Co-authored-by: Parthvi Vala <[email protected]>

* review

Co-authored-by: Parthvi Vala <[email protected]>

docs/website/versioned_docs/version-3.0.0/command-reference/deploy.md

@@ -64,3 +64,10 @@ components:
                 - name: main
                   image: {{CONTAINER_IMAGE}}
 ```
+
+## Sustituting variables
+
+The Devfile can define variables to make the Devfile parameterizable. The Devfile can define values for these variables, and you 
+can override the values for variables from the command line when running `odo deploy`, using the `--var` and `--var-file` options.
+
+See [Sustituting variables in odo dev](dev.md#sustituting-variables) for more information.

docs/website/versioned_docs/version-3.0.0/command-reference/dev.md

@@ -43,6 +43,55 @@ In the above example, three things have happened:
 You can press Ctrl-c at any time to terminate the development session. The command can take a few moment to terminate, as it
 will first delete all resources deployed into the cluster for this session before terminating.
 
+### Sustituting variables
+
+The Devfile can define variables to make the Devfile parameterizable. The Devfile can define values for these variables, and you 
+can override the values for variables from the command line when running `odo dev`, using the `--var` and `--var-file` options.
+
+The `--var` option is a repeatable option that takes a `variable=value` pair, where `=value` is optional. If the `=value` is omitted, the value is extracted from the environment variable named `variable`. In this case, if the environment variable with this name is not defined, the value defined into the Devfile will be used.
+
+The `--var-file` option takes a filename as argument. The file contains a line-separated list of `variable=value` pairs, with the same behaviour as before.  
+
+Note that the values passed with the `--var` option overrides the values obtained with the `--var-file` option.
+
+#### Examples
+
+Considering the Devfile contains this `variables` field:
+
+```
+variables:
+  USER: anonymous
+  DEBUG: false
+```
+
+
+This command will override the `USER` Devfile variable with the value of the `USER` environment variable, if it is defined.
+It will also override the value of the `DEBUG` Devfile variable with the `true` value.
+
+```shell
+$ odo dev --var USER --var DEBUG=true
+```
+
+If you create a file `config.vars` containing:
+
+```
+USER
+DEBUG=true
+```
+
+The following command will have the same behaviour as the previous one:
+
+```shell
+$ odo dev --var-file config.vars
+```
+
+The following command will override the `USER` Devfile variable with the `john` value:
+
+
+```shell
+$ odo dev --var USER=john --var-file config.vars
+```
+
 ## Devfile (Advanced Usage)
 
 ### Devfile Overview

go.mod

@@ -7,8 +7,8 @@ require (
 	github.com/Netflix/go-expect v0.0.0-20201125194554-85d881c3777e
 	github.com/Xuanwo/go-locale v1.0.0
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/devfile/api/v2 v2.0.0-20220117162434-6e6e6a8bc14c
-	github.com/devfile/library v1.2.1-0.20220217161036-0f5995513e92
+	github.com/devfile/api/v2 v2.0.0-20220309195345-48ebbf1e51cf
+	github.com/devfile/library v1.2.1-0.20220602130922-85a4805bd59c
 	github.com/devfile/registry-support/index/generator v0.0.0-20220222194908-7a90a4214f3e
 	github.com/devfile/registry-support/registry-library v0.0.0-20220504150710-21de53798172
 	github.com/fatih/color v1.13.0

go.sum

@@ -353,10 +353,14 @@ github.com/devfile/api/v2 v2.0.0-20210910153124-da620cd1a7a1/go.mod h1:kLX/nW93g
 github.com/devfile/api/v2 v2.0.0-20211021164004-dabee4e633ed/go.mod h1:d99eTN6QxgzihOOFyOZA+VpUyD4Q1pYRYHZ/ci9J96Q=
 github.com/devfile/api/v2 v2.0.0-20220117162434-6e6e6a8bc14c h1:sjghKUov/WT71dBreHYQcTgQctxHT/p4uAhUFdGQfSU=
 github.com/devfile/api/v2 v2.0.0-20220117162434-6e6e6a8bc14c/go.mod h1:d99eTN6QxgzihOOFyOZA+VpUyD4Q1pYRYHZ/ci9J96Q=
+github.com/devfile/api/v2 v2.0.0-20220309195345-48ebbf1e51cf h1:FkwAOQtepscB5B0j++9S/eoicXj707MaP5HPIScz0sA=
+github.com/devfile/api/v2 v2.0.0-20220309195345-48ebbf1e51cf/go.mod h1:kLX/nW93gigOHXK3NLeJL2fSS/sgEe+OHu8bo3aoOi4=
 github.com/devfile/library v1.1.1-0.20210910214722-7c5ff63711ec/go.mod h1:svPWwWb+BP15SXCHl0dyOeE4Sohrjl5a2BaOzc/riLc=
 github.com/devfile/library v1.2.1-0.20211104222135-49d635cb492f/go.mod h1:uFZZdTuRqA68FVe/JoJHP92CgINyQkyWnM2Qyiim+50=
 github.com/devfile/library v1.2.1-0.20220217161036-0f5995513e92 h1:RIrd0pBAET9vLHEZGyaG1PSjp/lJXa+CZfuWiih2p6g=
 github.com/devfile/library v1.2.1-0.20220217161036-0f5995513e92/go.mod h1:GSPfJaBg0+bBjBHbwBE5aerJLH6tWGQu2q2rHYd9czM=
+github.com/devfile/library v1.2.1-0.20220602130922-85a4805bd59c h1:Puqg/NJ6pP1LPFxPYBX9c0Oh0Ttu5PVdNJL5WR/CRTk=
+github.com/devfile/library v1.2.1-0.20220602130922-85a4805bd59c/go.mod h1:kpkIA9YI7gkfglRDsKJ3n5QgVyyBmnasr2U2djYwg6s=
 github.com/devfile/registry-support/index/generator v0.0.0-20211012185733-0a73f866043f h1:fKNUmoOPh7yAs69uMRZWHvev+m3e7T4jBL/hOXZB9ys=
 github.com/devfile/registry-support/index/generator v0.0.0-20211012185733-0a73f866043f/go.mod h1:bLGagbW2SFn7jo5+kUPlCMehIGqWkRtLKc5O0OyJMJM=
 github.com/devfile/registry-support/index/generator v0.0.0-20220222194908-7a90a4214f3e h1:3WAjUoyAmCBzUpx+sO0dSgkH74uSW1y886wGbojD2D8=

pkg/dev/dev.go

@@ -56,7 +56,7 @@ func (o *DevClient) Start(devfileObj parser.DevfileObj, platformContext kubernet
 	return nil
 }
 
-func (o *DevClient) Watch(devfileObj parser.DevfileObj, path string, ignorePaths []string, out io.Writer, h Handler, ctx context.Context, debug bool) error {
+func (o *DevClient) Watch(devfileObj parser.DevfileObj, path string, ignorePaths []string, out io.Writer, h Handler, ctx context.Context, debug bool, variables map[string]string) error {
 	envSpecificInfo, err := envinfo.NewEnvSpecificInfo(path)
 	if err != nil {
 		return err
@@ -72,6 +72,7 @@ func (o *DevClient) Watch(devfileObj parser.DevfileObj, path string, ignorePaths
 		InitialDevfileObj:   devfileObj,
 		Debug:               debug,
 		DebugPort:           envSpecificInfo.GetDebugPort(),
+		Variables:           variables,
 	}
 
 	return o.watchClient.WatchAndPush(out, watchParameters, ctx)

pkg/dev/interface.go

@@ -20,7 +20,7 @@ type Client interface {
 	// It logs messages to out and uses the Handler h to perform push operation when anything changes in path.
 	// It uses devfileObj to notify user to restart odo dev if they change endpoint information in the devfile.
 	// If debug is true, the debug command will be started after a sync, or the run command by default
-	Watch(devfileObj parser.DevfileObj, path string, ignorePaths []string, out io.Writer, h Handler, ctx context.Context, debug bool) error
+	Watch(devfileObj parser.DevfileObj, path string, ignorePaths []string, out io.Writer, h Handler, ctx context.Context, debug bool, variables map[string]string) error
 }
 
 type Handler interface {

pkg/devfile/devfile.go

@@ -47,6 +47,16 @@ func ParseAndValidateFromFile(devfilePath string) (parser.DevfileObj, error) {
 	return parseDevfile(parser.ParserArgs{Path: devfilePath})
 }
 
+// ParseAndValidateFromFileWithVariables reads, parses and validates  devfile from a file
+// variables are used to override devfile variables
+// if there are warning it logs them on stdout
+func ParseAndValidateFromFileWithVariables(devfilePath string, variables map[string]string) (parser.DevfileObj, error) {
+	return parseDevfile(parser.ParserArgs{
+		Path:              devfilePath,
+		ExternalVariables: variables,
+	})
+}
+
 func variableWarning(section string, variable string, messages []string) string {
 	quotedVars := []string{}
 	for _, v := range messages {

pkg/odo/cli/deploy/deploy.go

@@ -16,6 +16,7 @@ import (
 	"github.com/redhat-developer/odo/pkg/odo/genericclioptions/clientset"
 	odoutil "github.com/redhat-developer/odo/pkg/odo/util"
 	scontext "github.com/redhat-developer/odo/pkg/segment/context"
+	"github.com/redhat-developer/odo/pkg/vars"
 	"github.com/redhat-developer/odo/pkg/version"
 
 	"os"
@@ -36,6 +37,13 @@ type DeployOptions struct {
 	// Clients
 	clientset *clientset.Clientset
 
+	// Flags
+	varFileFlag string
+	varsFlag    []string
+
+	// Variables to override Devfile variables
+	variables map[string]string
+
 	// working directory
 	contextDir string
 }
@@ -85,7 +93,12 @@ func (o *DeployOptions) Complete(cmdline cmdline.Cmdline, args []string) (err er
 		return err
 	}
 
-	o.Context, err = genericclioptions.New(genericclioptions.NewCreateParameters(cmdline).NeedDevfile(o.contextDir).CreateAppIfNeeded())
+	o.variables, err = vars.GetVariables(o.clientset.FS, o.varFileFlag, o.varsFlag, os.LookupEnv)
+	if err != nil {
+		return err
+	}
+
+	o.Context, err = genericclioptions.New(genericclioptions.NewCreateParameters(cmdline).NeedDevfile(o.contextDir).WithVariables(o.variables).CreateAppIfNeeded())
 	if err != nil {
 		return err
 	}
@@ -161,7 +174,9 @@ func NewCmdDeploy(name, fullName string) *cobra.Command {
 			genericclioptions.GenericRun(o, cmd, args)
 		},
 	}
-	clientset.Add(deployCmd, clientset.INIT, clientset.DEPLOY)
+	deployCmd.Flags().StringArrayVar(&o.varsFlag, "var", []string{}, "Variable to override Devfile variable and variables in var-file")
+	deployCmd.Flags().StringVar(&o.varFileFlag, "var-file", "", "File containing variables to override Devfile variables")
+	clientset.Add(deployCmd, clientset.INIT, clientset.DEPLOY, clientset.FILESYSTEM)
 
 	// Add a defined annotation in order to appear in the help menu
 	deployCmd.Annotations["command"] = "main"

pkg/odo/cli/dev/dev.go

@@ -32,6 +32,7 @@ import (
 	odoutil "github.com/redhat-developer/odo/pkg/odo/util"
 	scontext "github.com/redhat-developer/odo/pkg/segment/context"
 	"github.com/redhat-developer/odo/pkg/util"
+	"github.com/redhat-developer/odo/pkg/vars"
 	"github.com/redhat-developer/odo/pkg/version"
 	"github.com/redhat-developer/odo/pkg/watch"
 )
@@ -65,6 +66,11 @@ type DevOptions struct {
 	noWatchFlag     bool
 	randomPortsFlag bool
 	debugFlag       bool
+	varFileFlag     string
+	varsFlag        []string
+
+	// Variables to override Devfile variables
+	variables map[string]string
 }
 
 type Handler struct{}
@@ -122,7 +128,12 @@ func (o *DevOptions) Complete(cmdline cmdline.Cmdline, args []string) error {
 		return err
 	}
 
-	o.Context, err = genericclioptions.New(genericclioptions.NewCreateParameters(cmdline).NeedDevfile(""))
+	o.variables, err = vars.GetVariables(o.clientset.FS, o.varFileFlag, o.varsFlag, os.LookupEnv)
+	if err != nil {
+		return err
+	}
+
+	o.Context, err = genericclioptions.New(genericclioptions.NewCreateParameters(cmdline).NeedDevfile("").WithVariables(o.variables))
 	if err != nil {
 		return fmt.Errorf("unable to create context: %v", err)
 	}
@@ -257,7 +268,7 @@ func (o *DevOptions) Run(ctx context.Context) (err error) {
 		err = o.clientset.WatchClient.CleanupDevResources(devFileObj, log.GetStdout())
 	} else {
 		d := Handler{}
-		err = o.clientset.DevClient.Watch(devFileObj, path, o.ignorePaths, o.out, &d, o.ctx, o.debugFlag)
+		err = o.clientset.DevClient.Watch(devFileObj, path, o.ignorePaths, o.out, &d, o.ctx, o.debugFlag, o.variables)
 	}
 	return err
 }
@@ -280,7 +291,7 @@ func (o *Handler) RegenerateAdapterAndPush(pushParams common.PushParameters, wat
 }
 
 func regenerateComponentAdapterFromWatchParams(parameters watch.WatchParameters) (common.ComponentAdapter, error) {
-	devObj, err := ododevfile.ParseAndValidateFromFile(location.DevfileLocation(""))
+	devObj, err := ododevfile.ParseAndValidateFromFileWithVariables(location.DevfileLocation(""), parameters.Variables)
 	if err != nil {
 		return nil, err
 	}
@@ -321,8 +332,9 @@ It forwards endpoints with exposure values 'public' or 'internal' to a port on l
 	devCmd.Flags().BoolVar(&o.noWatchFlag, "no-watch", false, "Do not watch for file changes")
 	devCmd.Flags().BoolVar(&o.randomPortsFlag, "random-ports", false, "Assign random ports to redirected ports")
 	devCmd.Flags().BoolVar(&o.debugFlag, "debug", false, "Execute the debug command within the component")
-
-	clientset.Add(devCmd, clientset.DEV, clientset.INIT, clientset.KUBERNETES, clientset.STATE)
+	devCmd.Flags().StringArrayVar(&o.varsFlag, "var", []string{}, "Variable to override Devfile variable and variables in var-file")
+	devCmd.Flags().StringVar(&o.varFileFlag, "var-file", "", "File containing variables to override Devfile variables")
+	clientset.Add(devCmd, clientset.DEV, clientset.INIT, clientset.KUBERNETES, clientset.STATE, clientset.FILESYSTEM)
 	// Add a defined annotation in order to appear in the help menu
 	devCmd.Annotations["command"] = "main"
 	devCmd.SetUsageTemplate(odoutil.CmdUsageTemplate)

pkg/odo/genericclioptions/context.go

@@ -60,6 +60,8 @@ type CreateParameters struct {
 	devfile          bool
 	offline          bool
 	appIfNeeded      bool
+	// variables override devfile variables
+	variables map[string]string
 }
 
 func NewCreateParameters(cmdline cmdline.Cmdline) CreateParameters {
@@ -82,6 +84,11 @@ func (o CreateParameters) CreateAppIfNeeded() CreateParameters {
 	return o
 }
 
+func (o CreateParameters) WithVariables(variables map[string]string) CreateParameters {
+	o.variables = variables
+	return o
+}
+
 // New creates a context based on the given parameters
 func New(parameters CreateParameters) (*Context, error) {
 	ctx := internalCxt{}
@@ -124,7 +131,7 @@ func New(parameters CreateParameters) (*Context, error) {
 		isDevfile := odoutil.CheckPathExists(ctx.devfilePath)
 		if isDevfile {
 			// Parse devfile and validate
-			devObj, err := devfile.ParseAndValidateFromFile(ctx.devfilePath)
+			devObj, err := devfile.ParseAndValidateFromFileWithVariables(ctx.devfilePath, parameters.variables)
 			if err != nil {
 				return nil, fmt.Errorf("failed to parse the devfile %s, with error: %s", ctx.devfilePath, err)
 			}

pkg/vars/doc.go

@@ -0,0 +1,2 @@
+// The var package analyzes variables defined in env file and as flags
+package vars

pkg/vars/errors.go

@@ -0,0 +1,15 @@
+package vars
+
+import "fmt"
+
+type ErrBadKey struct {
+	msg string
+}
+
+func NewErrBadKey(msg string) ErrBadKey {
+	return ErrBadKey{msg: msg}
+}
+
+func (e ErrBadKey) Error() string {
+	return fmt.Sprintf("poorly formatted environment: %s", e.msg)
+}