Merge pull request #21 from yolo3301/mig-merge

Merge from upstream

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
   test:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
@@ -34,7 +34,7 @@ jobs:
   swarm-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
@@ -61,7 +61,7 @@ jobs:
   dcos-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
@@ -70,6 +70,7 @@ jobs:
           echo 'export TIMEOUT=20m' >> $BASH_ENV
           echo 'export CLUSTER_DEFINITION=examples/dcos.json' >> $BASH_ENV
           echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_DCOS}' >> $BASH_ENV
+          echo 'export RETAIN_SSH=false' >> $BASH_ENV
           echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_DCOS}' >> $BASH_ENV
           echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_DCOS}' >> $BASH_ENV
       - run:
@@ -83,7 +84,7 @@ jobs:
   swarmmode-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
@@ -110,7 +111,7 @@ jobs:
   k8s-1.7-release-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
@@ -120,12 +121,17 @@ jobs:
           echo 'export ORCHESTRATOR_RELEASE=1.7' >> $BASH_ENV
           echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
           echo 'export CREATE_VNET=true' >> $BASH_ENV
+          echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV
+          echo 'export RETAIN_SSH=false' >> $BASH_ENV
           echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV
+      - run:
+          name: compile
+          command: make build-binary
       - run:
           name: ginkgo k8s e2e tests
-          command: make build-binary test-kubernetes
+          command: make test-kubernetes
           no_output_timeout: "30m"
       - store_artifacts:
           path: /go/src/github.com/Azure/acs-engine/_logs
@@ -134,21 +140,26 @@ jobs:
   k8s-windows-1.7-release-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
       - checkout
       - run: |
           echo 'export TIMEOUT=30m' >> $BASH_ENV
           echo 'export ORCHESTRATOR_RELEASE=1.7' >> $BASH_ENV
-          echo 'export CLUSTER_DEFINITION=examples/windows/kubernetes.json' >> $BASH_ENV
+          echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/windows/definition.json' >> $BASH_ENV
+          echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV
+          echo 'export RETAIN_SSH=false' >> $BASH_ENV
           echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV
+      - run:
+          name: compile
+          command: make build-binary
       - run:
           name: ginkgo k8s windows e2e tests
-          command: make build-binary test-kubernetes
+          command: make test-kubernetes
           no_output_timeout: "30m"
       - store_artifacts:
           path: /go/src/github.com/Azure/acs-engine/_logs
@@ -157,7 +168,7 @@ jobs:
   k8s-1.9-release-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
@@ -167,12 +178,17 @@ jobs:
           echo 'export ORCHESTRATOR_RELEASE=1.9' >> $BASH_ENV
           echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
           echo 'export CREATE_VNET=true' >> $BASH_ENV
+          echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV
+          echo 'export RETAIN_SSH=false' >> $BASH_ENV
           echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV
+      - run:
+          name: compile
+          command: make build-binary
       - run:
           name: ginkgo k8s e2e tests
-          command: make build-binary test-kubernetes
+          command: make test-kubernetes
           no_output_timeout: "30m"
       - store_artifacts:
           path: /go/src/github.com/Azure/acs-engine/_logs
@@ -181,7 +197,7 @@ jobs:
   k8s-1.8-release-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
@@ -191,12 +207,17 @@ jobs:
           echo 'export ORCHESTRATOR_RELEASE=1.8' >> $BASH_ENV
           echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/release/default/definition.json' >> $BASH_ENV
           echo 'export CREATE_VNET=true' >> $BASH_ENV
+          echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV
+          echo 'export RETAIN_SSH=false' >> $BASH_ENV
           echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV
+      - run:
+          name: compile
+          command: make build-binary
       - run:
           name: ginkgo k8s e2e tests
-          command: make build-binary test-kubernetes
+          command: make test-kubernetes
           no_output_timeout: "30m"
       - store_artifacts:
           path: /go/src/github.com/Azure/acs-engine/_logs
@@ -205,21 +226,26 @@ jobs:
   k8s-windows-1.8-release-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
       - checkout
       - run: |
           echo 'export TIMEOUT=30m' >> $BASH_ENV
           echo 'export ORCHESTRATOR_RELEASE=1.8' >> $BASH_ENV
-          echo 'export CLUSTER_DEFINITION=examples/windows/kubernetes.json' >> $BASH_ENV
+          echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/windows/definition.json' >> $BASH_ENV
+          echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV
+          echo 'export RETAIN_SSH=false' >> $BASH_ENV
           echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV
+      - run:
+          name: compile
+          command: make build-binary
       - run:
           name: ginkgo k8s windows e2e tests
-          command: make build-binary test-kubernetes
+          command: make test-kubernetes
           no_output_timeout: "30m"
       - store_artifacts:
           path: /go/src/github.com/Azure/acs-engine/_logs
@@ -228,21 +254,26 @@ jobs:
   k8s-windows-1.9-release-e2e:
     working_directory: /go/src/github.com/Azure/acs-engine
     docker:
-      - image: quay.io/deis/go-dev:v1.2.0
+      - image: quay.io/deis/go-dev:v1.8.1
         environment:
           GOPATH: /go
     steps:
       - checkout
       - run: |
           echo 'export TIMEOUT=30m' >> $BASH_ENV
           echo 'export ORCHESTRATOR_RELEASE=1.9' >> $BASH_ENV
-          echo 'export CLUSTER_DEFINITION=examples/windows/kubernetes.json' >> $BASH_ENV
+          echo 'export CLUSTER_DEFINITION=examples/e2e-tests/kubernetes/windows/definition.json' >> $BASH_ENV
+          echo 'export CLEANUP_ON_EXIT=false' >> $BASH_ENV
+          echo 'export RETAIN_SSH=false' >> $BASH_ENV
           echo 'export SUBSCRIPTION_ID=${SUBSCRIPTION_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_ID=${SERVICE_PRINCIPAL_CLIENT_ID_E2E_KUBERNETES}' >> $BASH_ENV
           echo 'export CLIENT_SECRET=${SERVICE_PRINCIPAL_CLIENT_SECRET_E2E_KUBERNETES}' >> $BASH_ENV
+      - run:
+          name: compile
+          command: make build-binary
       - run:
           name: ginkgo k8s windows e2e tests
-          command: make build-binary test-kubernetes
+          command: make test-kubernetes
           no_output_timeout: "30m"
       - store_artifacts:
           path: /go/src/github.com/Azure/acs-engine/_logs

.gitignore

@@ -1,5 +1,6 @@
 acs-engine.exe
 acs-engine
+debug
 _output/
 _input/
 .vscode

Makefile

@@ -25,7 +25,7 @@ GITTAG := $(VERSION_SHORT)
 endif
 
 REPO_PATH := github.com/Azure/acs-engine
-DEV_ENV_IMAGE := quay.io/deis/go-dev:v1.2.0
+DEV_ENV_IMAGE := quay.io/deis/go-dev:v1.8.1
 DEV_ENV_WORK_DIR := /go/src/${REPO_PATH}
 DEV_ENV_OPTS := --rm -v ${CURDIR}:${DEV_ENV_WORK_DIR} -w ${DEV_ENV_WORK_DIR} ${DEV_ENV_VARS}
 DEV_ENV_CMD := docker run ${DEV_ENV_OPTS} ${DEV_ENV_IMAGE}

cmd/deploy.go

@@ -111,14 +111,20 @@ func (dc *deployCmd) validate(cmd *cobra.Command, args []string) error {
 			Locale: dc.locale,
 		},
 	}
+
+	if dc.location == "" {
+		return fmt.Errorf(fmt.Sprintf("--location must be specified"))
+	}
 	// skip validating the model fields for now
 	dc.containerService, dc.apiVersion, err = apiloader.LoadContainerServiceFromFile(dc.apimodelPath, false, false, nil)
 	if err != nil {
 		return fmt.Errorf(fmt.Sprintf("error parsing the api model: %s", err.Error()))
 	}
 
-	if dc.location == "" {
-		return fmt.Errorf(fmt.Sprintf("--location must be specified"))
+	if dc.containerService.Location == "" {
+		dc.containerService.Location = dc.location
+	} else if dc.containerService.Location != dc.location {
+		return fmt.Errorf(fmt.Sprintf("--location does not match api model location"))
 	}
 
 	dc.client, err = dc.authArgs.getClient()
@@ -154,15 +160,13 @@ func autofillApimodel(dc *deployCmd) {
 		if dc.dnsPrefix == "" {
 			log.Fatalf("apimodel: missing masterProfile.dnsPrefix and --dns-prefix was not specified")
 		}
+		log.Warnf("apimodel: missing masterProfile.dnsPrefix will use %q", dc.dnsPrefix)
+		dc.containerService.Properties.MasterProfile.DNSPrefix = dc.dnsPrefix
+	}
 
-		dnsPrefix := dc.dnsPrefix
-		if dc.autoSuffix {
-			suffix := strconv.FormatInt(time.Now().Unix(), 16)
-			dnsPrefix = dnsPrefix + "-" + suffix
-		}
-
-		log.Warnf("apimodel: missing masterProfile.dnsPrefix will use %q", dnsPrefix)
-		dc.containerService.Properties.MasterProfile.DNSPrefix = dnsPrefix
+	if dc.autoSuffix {
+		suffix := strconv.FormatInt(time.Now().Unix(), 16)
+		dc.containerService.Properties.MasterProfile.DNSPrefix += "-" + suffix
 	}
 
 	if dc.outputDirectory == "" {
@@ -217,14 +221,11 @@ func autofillApimodel(dc *deployCmd) {
 			log.Warnf("created application with applicationID (%s) and servicePrincipalObjectID (%s).", applicationID, servicePrincipalObjectID)
 
 			log.Warnln("apimodel: ServicePrincipalProfile was empty, assigning role to application...")
-			for {
-				err = dc.client.CreateRoleAssignmentSimple(dc.resourceGroup, servicePrincipalObjectID)
-				if err != nil {
-					log.Debugf("Failed to create role assignment (will retry): %q", err)
-					time.Sleep(3 * time.Second)
-					continue
-				}
-				break
+
+			err = dc.client.CreateRoleAssignmentSimple(dc.resourceGroup, servicePrincipalObjectID)
+			if err != nil {
+				log.Fatalf("apimodel: could not create or assign ServicePrincipal: %q", err)
+
 			}
 
 			dc.containerService.Properties.ServicePrincipalProfile = &api.ServicePrincipalProfile{
@@ -256,7 +257,7 @@ func (dc *deployCmd) run() error {
 		log.Fatalln("failed to initialize template generator: %s", err.Error())
 	}
 
-	template, parameters, certsgenerated, err := templateGenerator.GenerateTemplate(dc.containerService, acsengine.DefaultGeneratorCode)
+	template, parameters, certsgenerated, err := templateGenerator.GenerateTemplate(dc.containerService, acsengine.DefaultGeneratorCode, false)
 	if err != nil {
 		log.Fatalf("error generating template %s: %s", dc.apimodelPath, err.Error())
 		os.Exit(1)

cmd/deploy_test.go

@@ -26,9 +26,27 @@ const ExampleAPIModel = `{
 }
 `
 
+const ExampleAPIModelWithDNSPrefix = `{
+	"apiVersion": "vlabs",
+	"properties": {
+		  "orchestratorProfile": { "orchestratorType": "Kubernetes", "kubernetesConfig": { "useManagedIdentity": %s, "etcdVersion" : "2.3.8" } },
+	  "masterProfile": { "count": 1, "dnsPrefix": "mytestcluster", "vmSize": "Standard_D2_v2" },
+	  "agentPoolProfiles": [ { "name": "linuxpool1", "count": 2, "vmSize": "Standard_D2_v2", "availabilityProfile": "AvailabilitySet" } ],
+	  "windowsProfile": { "adminUsername": "azureuser", "adminPassword": "replacepassword1234$" },
+	  "linuxProfile": { "adminUsername": "azureuser", "ssh": { "publicKeys": [ { "keyData": "" } ] }
+	  },
+	  "servicePrincipalProfile": { "clientId": "%s", "secret": "%s" }
+	}
+  }
+  `
+
 func getExampleAPIModel(useManagedIdentity bool, clientID, clientSecret string) string {
+	return getAPIModel(ExampleAPIModel, useManagedIdentity, clientID, clientSecret)
+}
+
+func getAPIModel(baseAPIModel string, useManagedIdentity bool, clientID, clientSecret string) string {
 	return fmt.Sprintf(
-		ExampleAPIModel,
+		baseAPIModel,
 		strconv.FormatBool(useManagedIdentity),
 		clientID,
 		clientSecret)
@@ -46,6 +64,36 @@ func TestAutofillApimodelAllowsPrespecifiedCreds(t *testing.T) {
 	testAutodeployCredentialHandling(t, false, "clientID", "clientSecret")
 }
 
+func TestAutoSufixWithDnsPrefixInApiModel(t *testing.T) {
+	apiloader := &api.Apiloader{
+		Translator: nil,
+	}
+
+	apimodel := getAPIModel(ExampleAPIModelWithDNSPrefix, false, "clientID", "clientSecret")
+	cs, ver, err := apiloader.DeserializeContainerService([]byte(apimodel), false, false, nil)
+	if err != nil {
+		t.Fatalf("unexpected error deserializing the example apimodel: %s", err)
+	}
+	deployCmd := &deployCmd{
+		apimodelPath:     "./this/is/unused.json",
+		outputDirectory:  "_test_output",
+		location:         "westus",
+		autoSuffix:       true,
+		containerService: cs,
+		apiVersion:       ver,
+
+		client: &armhelpers.MockACSEngineClient{},
+	}
+	autofillApimodel(deployCmd)
+
+	defer os.RemoveAll(deployCmd.outputDirectory)
+
+	if deployCmd.containerService.Properties.MasterProfile.DNSPrefix == "mytestcluster" {
+		t.Fatalf("expected %s-{timestampsuffix} but got %s", "mytestcluster", deployCmd.containerService.Properties.MasterProfile.DNSPrefix)
+	}
+
+}
+
 func testAutodeployCredentialHandling(t *testing.T, useManagedIdentity bool, clientID, clientSecret string) {
 	apiloader := &api.Apiloader{
 		Translator: nil,

cmd/generate.go

@@ -144,7 +144,7 @@ func (gc *generateCmd) run() error {
 		log.Fatalln("failed to initialize template generator: %s", err.Error())
 	}
 
-	template, parameters, certsGenerated, err := templateGenerator.GenerateTemplate(gc.containerService, acsengine.DefaultGeneratorCode)
+	template, parameters, certsGenerated, err := templateGenerator.GenerateTemplate(gc.containerService, acsengine.DefaultGeneratorCode, false)
 	if err != nil {
 		log.Fatalf("error generating template %s: %s", gc.apimodelPath, err.Error())
 		os.Exit(1)

cmd/orchestrators_test.go

@@ -40,7 +40,7 @@ var _ = Describe("The orchestrators command", func() {
 	It("should succeed", func() {
 		command := &orchestratorsCmd{
 			orchestrator: "kubernetes",
-			version:      "1.7.12",
+			version:      "1.7.13",
 		}
 
 		err := command.run(nil, nil)