Create a --set flag for generate

cmd/generate.go

@@ -30,6 +30,7 @@ type generateCmd struct {
 	classicMode       bool
 	noPrettyPrint     bool
 	parametersOnly    bool
+	set               []string
 
 	// derived
 	containerService *api.ContainerService
@@ -48,6 +49,15 @@ func newGenerateCmd() *cobra.Command {
 			if err := gc.validate(cmd, args); err != nil {
 				log.Fatalf(fmt.Sprintf("error validating generateCmd: %s", err.Error()))
 			}
+
+			if err := gc.mergeAPIModel(); err != nil {
+				log.Fatalf(fmt.Sprintf("error merging API model in generateCmd: %s", err.Error()))
+			}
+
+			if err := gc.loadAPIModel(cmd, args); err != nil {
+				log.Fatalf(fmt.Sprintf("error loading API model in generateCmd: %s", err.Error()))
+			}
+
 			return gc.run()
 		},
 	}
@@ -57,6 +67,7 @@ func newGenerateCmd() *cobra.Command {
 	f.StringVar(&gc.outputDirectory, "output-directory", "", "output directory (derived from FQDN if absent)")
 	f.StringVar(&gc.caCertificatePath, "ca-certificate-path", "", "path to the CA certificate to use for Kubernetes PKI assets")
 	f.StringVar(&gc.caPrivateKeyPath, "ca-private-key-path", "", "path to the CA private key to use for Kubernetes PKI assets")
+	f.StringArrayVar(&gc.set, "set", []string{}, "set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	f.BoolVar(&gc.classicMode, "classic-mode", false, "enable classic parameters and outputs")
 	f.BoolVar(&gc.noPrettyPrint, "no-pretty-print", false, "skip pretty printing the output")
 	f.BoolVar(&gc.parametersOnly, "parameters-only", false, "only output parameters files")
@@ -65,8 +76,6 @@ func newGenerateCmd() *cobra.Command {
 }
 
 func (gc *generateCmd) validate(cmd *cobra.Command, args []string) error {
-	var caCertificateBytes []byte
-	var caKeyBytes []byte
 	var err error
 
 	gc.locale, err = i18n.LoadTranslations()
@@ -90,6 +99,34 @@ func (gc *generateCmd) validate(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf(fmt.Sprintf("specified api model does not exist (%s)", gc.apimodelPath))
 	}
 
+	return nil
+}
+
+func (gc *generateCmd) mergeAPIModel() error {
+	var err error
+
+	// if --set flag has been used
+	if gc.set != nil && len(gc.set) > 0 {
+		m := make(map[string]transform.APIModelValue)
+		transform.MapValues(m, gc.set)
+
+		// overrides the api model and generates a new file
+		gc.apimodelPath, err = transform.MergeValuesWithAPIModel(gc.apimodelPath, m)
+		if err != nil {
+			return fmt.Errorf(fmt.Sprintf("error merging --set values with the api model: %s", err.Error()))
+		}
+
+		log.Infoln(fmt.Sprintf("new api model file has been generated during merge: %s", gc.apimodelPath))
+	}
+
+	return nil
+}
+
+func (gc *generateCmd) loadAPIModel(cmd *cobra.Command, args []string) error {
+	var caCertificateBytes []byte
+	var caKeyBytes []byte
+	var err error
+
 	apiloader := &api.Apiloader{
 		Translator: &i18n.Translator{
 			Locale: gc.locale,
@@ -128,6 +165,7 @@ func (gc *generateCmd) validate(cmd *cobra.Command, args []string) error {
 		prop.CertificateProfile.CaCertificate = string(caCertificateBytes)
 		prop.CertificateProfile.CaPrivateKey = string(caKeyBytes)
 	}
+
 	return nil
 }
 

cmd/generate_test.go

@@ -7,9 +7,7 @@ import (
 )
 
 func TestGenerateCmdValidate(t *testing.T) {
-
 	g := &generateCmd{}
-
 	r := &cobra.Command{}
 
 	// validate cmd with 1 arg
@@ -37,3 +35,51 @@ func TestGenerateCmdValidate(t *testing.T) {
 	}
 
 }
+
+func TestGenerateCmdMergeAPIModel(t *testing.T) {
+	g := &generateCmd{}
+	g.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json"
+	err := g.mergeAPIModel()
+	if err != nil {
+		t.Fatalf("unexpected error calling mergeAPIModel with no --set flag defined: %s", err.Error())
+	}
+
+	g = &generateCmd{}
+	g.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json"
+	g.set = []string{"masterProfile.count=3,linuxProfile.adminUsername=testuser"}
+	err = g.mergeAPIModel()
+	if err != nil {
+		t.Fatalf("unexpected error calling mergeAPIModel with one --set flag: %s", err.Error())
+	}
+
+	g = &generateCmd{}
+	g.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json"
+	g.set = []string{"masterProfile.count=3", "linuxProfile.adminUsername=testuser"}
+	err = g.mergeAPIModel()
+	if err != nil {
+		t.Fatalf("unexpected error calling mergeAPIModel with multiple --set flags: %s", err.Error())
+	}
+
+	g = &generateCmd{}
+	g.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json"
+	g.set = []string{"agentPoolProfiles[0].count=1"}
+	err = g.mergeAPIModel()
+	if err != nil {
+		t.Fatalf("unexpected error calling mergeAPIModel with one --set flag to override an array property: %s", err.Error())
+	}
+}
+
+func TestGenerateCmdMLoadAPIModel(t *testing.T) {
+	g := &generateCmd{}
+	r := &cobra.Command{}
+
+	g.apimodelPath = "../pkg/acsengine/testdata/simple/kubernetes.json"
+	g.set = []string{"agentPoolProfiles[0].count=1"}
+
+	g.validate(r, []string{"../pkg/acsengine/testdata/simple/kubernetes.json"})
+	g.mergeAPIModel()
+	err := g.loadAPIModel(r, []string{"../pkg/acsengine/testdata/simple/kubernetes.json"})
+	if err != nil {
+		t.Fatalf("unexpected error loading api model: %s", err.Error())
+	}
+}

docs/kubernetes/deploy.md

@@ -98,12 +98,26 @@ Edit the [simple Kubernetes cluster definition](/examples/kubernetes.json) and f
 
 Optional: attach to an existing virtual network (VNET). Details [here](features.md#feat-custom-vnet)
 
+Note: you can then use the `--set` option of the generate command to override values from the cluster definition file directly in the command line (cf. [Step 4](deploy.md#step-4-generate-the-templates))
+
 ### Step 4: Generate the Templates
 
 The generate command takes a cluster definition and outputs a number of templates which describe your Kubernetes cluster. By default, `generate` will create a new directory named after your cluster nested in the `_output` directory. If my dnsPrefix was `larry` my cluster templates would be found in `_output/larry-`.
 
 Run `acs-engine generate examples/kubernetes.json`
 
+The generate command lets you override values from the cluster definition file without having to update the file. You can use the `--set` flag to do that:
+
+```bash
+acs-engine generate --set linuxProfile.adminUsername=myNewUsername,masterProfile.count=3 clusterdefinition.json
+```
+
+The `--set` flag only supports JSON properties under `properties`. You can also work with array, like the following:
+
+```bash
+acs-engine generate --set agentPoolProfiles[0].count=5,agentPoolProfiles[1].name=myPoolName clusterdefinition.json
+```
+
 ### Step 5: Submit your Templates to Azure Resource Manager (ARM)
 
 [Deploy the output azuredeploy.json and azuredeploy.parameters.json](../acsengine.md#deployment-usage)

glide.yaml

@@ -56,6 +56,8 @@ import:
   version: 366b072768b4e6d93c7de236464c0abe85d0b7c6
 - package: k8s.io/client-go
   version: ~4.0.0
+- package: github.com/Jeffail/gabs
+  version: 1.0
 testImport:
 # glide isn't able to mutually reconcile pinned versions of these deps
 - package: github.com/onsi/gomega

pkg/acsengine/transform/apimodel_merger.go

@@ -0,0 +1,119 @@
+package transform
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/Jeffail/gabs"
+	log "github.com/sirupsen/logrus"
+)
+
+// APIModelValue represents a value in the APIModel JSON file
+type APIModelValue struct {
+	stringValue   string
+	intValue      int64
+	arrayValue    bool
+	arrayIndex    int
+	arrayProperty string
+	arrayName     string
+}
+
+// MapValues converts an arraw of rwa ApiModel values (like ["masterProfile.count=4","linuxProfile.adminUsername=admin"]) to a map
+func MapValues(m map[string]APIModelValue, values []string) {
+	if values == nil || len(values) == 0 {
+		return
+	}
+
+	for _, value := range values {
+		splittedValues := strings.Split(value, ",")
+		if len(splittedValues) > 1 {
+			MapValues(m, splittedValues)
+		} else {
+			keyValueSplitted := strings.Split(value, "=")
+			key := keyValueSplitted[0]
+			stringValue := keyValueSplitted[1]
+
+			flagValue := APIModelValue{}
+
+			if asInteger, err := strconv.ParseInt(stringValue, 10, 64); err == nil {
+				flagValue.intValue = asInteger
+			} else {
+				flagValue.stringValue = stringValue
+			}
+
+			// use regex to find array[index].property pattern in the key
+			re := regexp.MustCompile(`(.*?)\[(.*?)\]\.(.*?)$`)
+			match := re.FindStringSubmatch(key)
+
+			// it's an array
+			if len(match) != 0 {
+				i, err := strconv.ParseInt(match[2], 10, 32)
+				if err != nil {
+					log.Warnln(fmt.Sprintf("array index is not specified for property %s", key))
+				} else {
+					arrayIndex := int(i)
+					flagValue.arrayValue = true
+					flagValue.arrayName = match[1]
+					flagValue.arrayIndex = arrayIndex
+					flagValue.arrayProperty = match[3]
+					m[key] = flagValue
+				}
+			} else {
+				m[key] = flagValue
+			}
+		}
+	}
+}
+
+// MergeValuesWithAPIModel takes the path to an ApiModel JSON file, loads it and merges it with the values in the map to another temp file
+func MergeValuesWithAPIModel(apiModelPath string, m map[string]APIModelValue) (string, error) {
+	// load the apiModel file from path
+	fileContent, err := ioutil.ReadFile(apiModelPath)
+	if err != nil {
+		return "", err
+	}
+
+	// parse the json from file content
+	jsonObj, err := gabs.ParseJSON(fileContent)
+	if err != nil {
+		return "", err
+	}
+
+	// update api model definition with each value in the map
+	for key, flagValue := range m {
+		// working on an array
+		if flagValue.arrayValue {
+			log.Infoln(fmt.Sprintf("--set flag array value detected. Name: %s, Index: %b, PropertyName: %s", flagValue.arrayName, flagValue.arrayIndex, flagValue.arrayProperty))
+			arrayValue := jsonObj.Path(fmt.Sprint("properties.", flagValue.arrayName))
+			if flagValue.stringValue != "" {
+				arrayValue.Index(flagValue.arrayIndex).SetP(flagValue.stringValue, flagValue.arrayProperty)
+			} else {
+				arrayValue.Index(flagValue.arrayIndex).SetP(flagValue.intValue, flagValue.arrayProperty)
+			}
+		} else {
+			if flagValue.stringValue != "" {
+				jsonObj.SetP(flagValue.stringValue, fmt.Sprint("properties.", key))
+			} else {
+				jsonObj.SetP(flagValue.intValue, fmt.Sprint("properties.", key))
+			}
+		}
+	}
+
+	// generate a new file
+	tmpFile, err := ioutil.TempFile("", "mergedApiModel")
+	if err != nil {
+		return "", err
+	}
+
+	tmpFileName := tmpFile.Name()
+	err = ioutil.WriteFile(tmpFileName, []byte(jsonObj.String()), os.ModeAppend)
+	if err != nil {
+		return "", err
+	}
+
+	return tmpFileName, nil
+}

pkg/acsengine/transform/apimodel_merger_test.go

@@ -0,0 +1,50 @@
+package transform
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/Jeffail/gabs"
+	. "github.com/onsi/gomega"
+)
+
+func TestAPIModelMergerMapValues(t *testing.T) {
+	RegisterTestingT(t)
+
+	m := make(map[string]APIModelValue)
+	values := []string{"masterProfile.count=5", "agentPoolProfiles[0].name=agentpool1", "linuxProfile.adminUsername=admin"}
+
+	MapValues(m, values)
+	Expect(m["masterProfile.count"].intValue).To(BeIdenticalTo(int64(5)))
+	Expect(m["agentPoolProfiles[0].name"].arrayValue).To(BeTrue())
+	Expect(m["agentPoolProfiles[0].name"].arrayIndex).To(BeIdenticalTo(0))
+	Expect(m["agentPoolProfiles[0].name"].arrayProperty).To(BeIdenticalTo("name"))
+	Expect(m["agentPoolProfiles[0].name"].arrayName).To(BeIdenticalTo("agentPoolProfiles"))
+	Expect(m["agentPoolProfiles[0].name"].stringValue).To(BeIdenticalTo("agentpool1"))
+	Expect(m["linuxProfile.adminUsername"].stringValue).To(BeIdenticalTo("admin"))
+}
+
+func TestMergeValuesWithAPIModel(t *testing.T) {
+	RegisterTestingT(t)
+
+	m := make(map[string]APIModelValue)
+	values := []string{"masterProfile.count=5", "agentPoolProfiles[0].name=agentpool1", "linuxProfile.adminUsername=admin"}
+
+	MapValues(m, values)
+	tmpFile, _ := MergeValuesWithAPIModel("../testdata/simple/kubernetes.json", m)
+
+	jsonFileContent, err := ioutil.ReadFile(tmpFile)
+	Expect(err).To(BeNil())
+
+	jsonAPIModel, err := gabs.ParseJSON(jsonFileContent)
+	Expect(err).To(BeNil())
+
+	masterProfileCount := jsonAPIModel.Path("properties.masterProfile.count").Data()
+	Expect(masterProfileCount).To(BeIdenticalTo(float64(5)))
+
+	adminUsername := jsonAPIModel.Path("properties.linuxProfile.adminUsername").Data()
+	Expect(adminUsername).To(BeIdenticalTo("admin"))
+
+	agentPoolProfileName := jsonAPIModel.Path("properties.agentPoolProfiles").Index(0).Path("name").Data().(string)
+	Expect(agentPoolProfileName).To(BeIdenticalTo("agentpool1"))
+}