Merge remote-tracking branch 'origin/master' into apim_authentication…

…_settings

.gitignore

@@ -35,9 +35,6 @@ website/vendor
 
 .env.sh
 
-# goenv version file
-.go-version
-
 # generated by the example acceptance tests
 examples/**/test.tf
 examples/**/test.tfvars

.go-version

@@ -0,0 +1 @@
+1.14.2

.teamcity/.gitignore

@@ -0,0 +1,2 @@
+target/
+*.iml

.teamcity/Makefile

@@ -0,0 +1,10 @@
+default: tools
+
+tools:
+	mvn -U dependency:sources
+
+test: validate
+	mvn test -DtestPackage=tests
+
+validate:
+	mvn teamcity-configs:generate

.teamcity/components/build_azure.kt

@@ -0,0 +1,26 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.ParametrizedWithType
+
+class ClientConfiguration(var clientId: String,
+                          var clientSecret: String,
+                          val subscriptionId : String,
+                          val tenantId : String,
+                          val altClientId: String,
+                          val altClientSecret: String) {
+}
+
+class LocationConfiguration(var primary : String, var secondary : String, var ternary : String, var rotate : Boolean) {
+}
+
+fun ParametrizedWithType.ConfigureAzureSpecificTestParameters(environment: String, config: ClientConfiguration, locationsForEnv: LocationConfiguration) {
+    hiddenPasswordVariable("env.ARM_CLIENT_ID", config.clientId, "The ID of the Service Principal used for Testing")
+    hiddenPasswordVariable("env.ARM_CLIENT_ID_ALT", config.altClientId, "The ID of the Alternate Service Principal used for Testing")
+    hiddenPasswordVariable("env.ARM_CLIENT_SECRET", config.clientSecret, "The Client Secret of the Service Principal used for Testing")
+    hiddenPasswordVariable("env.ARM_CLIENT_SECRET_ALT", config.altClientSecret, "The Client Secret of the Alternate Service Principal used for Testing")
+    hiddenVariable("env.ARM_ENVIRONMENT", environment, "The Azure Environment in which the tests are running")
+    hiddenVariable("env.ARM_PROVIDER_DYNAMIC_TEST", "%b".format(locationsForEnv.rotate), "Should tests rotate between the supported regions?")
+    hiddenPasswordVariable("env.ARM_SUBSCRIPTION_ID", config.subscriptionId, "The ID of the Azure Subscription used for Testing")
+    hiddenPasswordVariable("env.ARM_TENANT_ID", config.tenantId, "The ID of the Azure Tenant used for Testing")
+    hiddenVariable("env.ARM_TEST_LOCATION", locationsForEnv.primary, "The Primary region which should be used for testing")
+    hiddenVariable("env.ARM_TEST_LOCATION_ALT", locationsForEnv.secondary, "The Primary region which should be used for testing")
+    hiddenVariable("env.ARM_TEST_LOCATION_ALT2", locationsForEnv.ternary, "The Primary region which should be used for testing")
+}

.teamcity/components/build_components.kt

@@ -0,0 +1,84 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.GolangFeature
+import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.ScriptBuildStep
+import jetbrains.buildServer.configs.kotlin.v2019_2.triggers.schedule
+
+// NOTE: in time this could be pulled out into a separate Kotlin package
+
+// unfortunately TeamCity's Go Test Json parser appears to be broken
+// as such for the moment let's use the old method with a feature-flag
+const val useTeamCityGoTest = true
+
+fun BuildFeatures.Golang() {
+    if (useTeamCityGoTest) {
+        feature(GolangFeature {
+            testFormat = "json"
+        })
+    }
+}
+
+fun BuildSteps.ConfigureGoEnv() {
+    step(ScriptBuildStep {
+        name = "Configure Go Version"
+        scriptContent = "goenv install -s \$(goenv local) && goenv rehash"
+    })
+}
+
+fun BuildSteps.RunAcceptanceTests(providerName : String, packageName: String) {
+    if (useTeamCityGoTest) {
+        var servicePath = "./%s/internal/services/%s/...".format(providerName, packageName)
+        step(ScriptBuildStep {
+            name = "Run Tests"
+            scriptContent = "go test -v \"$servicePath\" -timeout=\"%TIMEOUT%h\" -test.parallel=\"%PARALLELISM%\" -run=\"%TEST_PREFIX%\" -json"
+        })
+    } else {
+        step(ScriptBuildStep {
+            name = "Install tombuildsstuff/teamcity-go-test-json"
+            scriptContent = "wget https://github.com/tombuildsstuff/teamcity-go-test-json/releases/download/v0.2.0/teamcity-go-test-json_linux_amd64 && chmod +x teamcity-go-test-json_linux_amd64"
+        })
+
+        var servicePath = "./%s/internal/services/%s/...".format(providerName, packageName)
+        step(ScriptBuildStep {
+            name = "Run Tests"
+            scriptContent = "GOFLAGS=\"-mod=vendor\" ./teamcity-go-test-json_linux_amd64 -scope \"$servicePath\" -prefix \"%TEST_PREFIX%\" -count=1 -parallelism=%PARALLELISM% -timeout %TIMEOUT%"
+        })
+    }
+}
+
+fun ParametrizedWithType.TerraformAcceptanceTestParameters(parallelism : Int, prefix : String, timeout: String) {
+    text("PARALLELISM", "%d".format(parallelism))
+    text("TEST_PREFIX", prefix)
+    text("TIMEOUT", timeout)
+}
+
+fun ParametrizedWithType.ReadOnlySettings() {
+    hiddenVariable("teamcity.ui.settings.readOnly", "true", "Requires build configurations be edited via Kotlin")
+}
+
+fun ParametrizedWithType.TerraformAcceptanceTestsFlag() {
+    hiddenVariable("env.TF_ACC", "1", "Set to a value to run the Acceptance Tests")
+}
+
+fun ParametrizedWithType.TerraformShouldPanicForSchemaErrors() {
+    hiddenVariable("env.TF_SCHEMA_PANIC_ON_ERROR", "1", "Panic if unknown/unmatched fields are set into the state")
+}
+
+fun ParametrizedWithType.hiddenVariable(name: String, value: String, description: String) {
+    text(name, value, "", description, ParameterDisplay.HIDDEN)
+}
+
+fun ParametrizedWithType.hiddenPasswordVariable(name: String, value: String, description: String) {
+    password(name, value, "", description, ParameterDisplay.HIDDEN)
+}
+
+fun Triggers.RunNightly(nightlyTestsEnabled: Boolean, startHour: Int) {
+    schedule{
+        enabled = nightlyTestsEnabled
+        branchFilter = "+:refs/heads/master"
+
+        schedulingPolicy = daily {
+            hour = startHour
+            timezone = "SERVER"
+        }
+    }
+}

.teamcity/components/build_config_pull_request.kt

@@ -0,0 +1,48 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+
+class pullRequest(displayName: String, environment: String) {
+    val displayName = displayName
+    val environment = environment
+
+    fun buildConfiguration(providerName : String) : BuildType {
+        return BuildType {
+            // TC needs a consistent ID for dynamically generated packages
+            id(uniqueID(providerName))
+
+            name = displayName
+
+            vcs {
+                root(providerRepository)
+                cleanCheckout = true
+            }
+
+            steps {
+                var packageName = "\"%SERVICES%\""
+
+                ConfigureGoEnv()
+                RunAcceptanceTests(providerName, packageName)
+            }
+
+            failureConditions {
+                errorMessage = true
+            }
+
+            features {
+                Golang()
+            }
+
+            params {
+                TerraformAcceptanceTestParameters(defaultParallelism, "TestAcc", "12")
+                TerraformAcceptanceTestsFlag()
+                TerraformShouldPanicForSchemaErrors()
+                ReadOnlySettings()
+
+                text("SERVICES", "portal")
+            }
+        }
+    }
+
+    fun uniqueID(provider : String) : String {
+        return "%s_PR_%s".format(provider.toUpperCase(), environment.toUpperCase())
+    }
+}

.teamcity/components/build_config_service.kt

@@ -0,0 +1,49 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.*
+
+class serviceDetails(name: String, displayName: String, environment: String) {
+    val packageName = name
+    val displayName = displayName
+    val environment = environment
+
+    fun buildConfiguration(providerName : String, nightlyTestsEnabled: Boolean, startHour: Int, parallelism: Int) : BuildType {
+        return BuildType {
+            // TC needs a consistent ID for dynamically generated packages
+            id(uniqueID(providerName))
+
+            name = "%s - Acceptance Tests".format(displayName)
+
+            vcs {
+                root(providerRepository)
+                cleanCheckout = true
+            }
+
+            steps {
+                ConfigureGoEnv()
+                RunAcceptanceTests(providerName, packageName)
+            }
+
+            failureConditions {
+                errorMessage = true
+            }
+
+            features {
+                Golang()
+            }
+
+            params {
+                TerraformAcceptanceTestParameters(parallelism, "TestAcc", "12")
+                TerraformAcceptanceTestsFlag()
+                TerraformShouldPanicForSchemaErrors()
+                ReadOnlySettings()
+            }
+
+            triggers {
+                RunNightly(nightlyTestsEnabled, startHour)
+            }
+        }
+    }
+
+    fun uniqueID(provider : String) : String {
+        return "%s_SERVICE_%s_%s".format(provider.toUpperCase(), environment.toUpperCase(), packageName.toUpperCase())
+    }
+}

.teamcity/components/generated/services.kt

@@ -0,0 +1,75 @@
+// NOTE: this is Generated from the Service Definitions - manual changes will be lost
+//       to re-generate this file, run 'make generate' in the root of the repository
+var services = mapOf(
+        "analysisservices" to "Analysis Services",
+        "apimanagement" to "API Management",
+        "appconfiguration" to "App Configuration",
+        "appplatform" to "App Platform",
+        "applicationinsights" to "Application Insights",
+        "authorization" to "Authorization",
+        "automation" to "Automation",
+        "batch" to "Batch",
+        "bot" to "Bot",
+        "cdn" to "CDN",
+        "cognitive" to "Cognitive Services",
+        "compute" to "Compute",
+        "containers" to "Container Services",
+        "cosmos" to "CosmosDB",
+        "costmanagement" to "Cost Management",
+        "customproviders" to "Custom Providers",
+        "databricks" to "DataBricks",
+        "datafactory" to "Data Factory",
+        "datalake" to "Data Lake",
+        "databasemigration" to "Database Migration",
+        "datashare" to "Data Share",
+        "devspace" to "DevSpaces",
+        "devtestlabs" to "Dev Test",
+        "dns" to "DNS",
+        "eventgrid" to "EventGrid",
+        "eventhub" to "EventHub",
+        "frontdoor" to "FrontDoor",
+        "hdinsight" to "HDInsight",
+        "healthcare" to "Health Care",
+        "iothub" to "IoT Hub",
+        "iotcentral" to "IoT Central",
+        "keyvault" to "KeyVault",
+        "kusto" to "Kusto",
+        "loganalytics" to "Log Analytics",
+        "logic" to "Logic",
+        "machinelearning" to "Machine Learning",
+        "maintenance" to "Maintenance",
+        "managedapplications" to "Managed Applications",
+        "managementgroup" to "Management Group",
+        "maps" to "Maps",
+        "mariadb" to "MariaDB",
+        "media" to "Media",
+        "mixedreality" to "Mixed Reality",
+        "monitor" to "Monitor",
+        "msi" to "Managed Service Identities",
+        "mssql" to "Microsoft SQL Server / SQL Azure",
+        "mysql" to "MySQL",
+        "netapp" to "NetApp",
+        "network" to "Network",
+        "notificationhub" to "Notification Hub",
+        "policy" to "Policy",
+        "portal" to "Portal",
+        "postgres" to "PostgreSQL",
+        "powerbi" to "PowerBI",
+        "privatedns" to "Private DNS",
+        "recoveryservices" to "Recovery Services",
+        "redis" to "Redis",
+        "relay" to "Relay",
+        "resource" to "Resources",
+        "search" to "Search",
+        "securitycenter" to "Security Center",
+        "sentinel" to "Sentinel",
+        "servicebus" to "ServiceBus",
+        "servicefabric" to "Service Fabric",
+        "signalr" to "SignalR",
+        "sql" to "SQL",
+        "storage" to "Storage",
+        "streamanalytics" to "Stream Analytics",
+        "subscription" to "Subscription",
+        "trafficmanager" to "Traffic Manager",
+        "web" to "Web"
+)

.teamcity/components/project.kt

@@ -0,0 +1,52 @@
+import jetbrains.buildServer.configs.kotlin.v2019_2.BuildType
+import jetbrains.buildServer.configs.kotlin.v2019_2.Project
+
+const val providerName = "azurerm"
+
+fun AzureRM(environment: String, configuration : ClientConfiguration) : Project {
+    return Project{
+        vcsRoot(providerRepository)
+
+        var pullRequestBuildConfig = pullRequestBuildConfiguration(environment, configuration)
+        buildType(pullRequestBuildConfig)
+
+        var buildConfigs = buildConfigurationsForServices(services, providerName, environment, configuration)
+        buildConfigs.forEach { buildConfiguration ->
+            buildType(buildConfiguration)
+        }
+    }
+}
+
+fun buildConfigurationsForServices(services: Map<String, String>, providerName : String, environment: String, config : ClientConfiguration): List<BuildType> {
+    var list = ArrayList<BuildType>()
+    var locationsForEnv = locations.get(environment)!!
+
+    services.forEach { (serviceName, displayName) ->
+        // TODO: overriding locations
+        var defaultTestConfig = testConfiguration(defaultParallelism, defaultStartHour)
+        var testConfig = serviceTestConfigurationOverrides.getOrDefault(serviceName, defaultTestConfig)
+        var runNightly = runNightly.getOrDefault(environment, false)
+
+        var service = serviceDetails(serviceName, displayName, environment)
+        var buildConfig = service.buildConfiguration(providerName, runNightly, testConfig.startHour, testConfig.parallelism)
+
+        buildConfig.params.ConfigureAzureSpecificTestParameters(environment, config, locationsForEnv)
+
+        list.add(buildConfig)
+    }
+
+    return list
+}
+
+fun pullRequestBuildConfiguration(environment: String, configuration: ClientConfiguration) : BuildType {
+    var locationsForEnv = locations.get(environment)!!
+    var pullRequest = pullRequest("! Run Pull Request", environment)
+    var buildConfiguration = pullRequest.buildConfiguration(providerName)
+    buildConfiguration.params.ConfigureAzureSpecificTestParameters(environment, configuration, locationsForEnv)
+    return buildConfiguration
+}
+
+class testConfiguration(parallelism: Int, startHour: Int) {
+    var parallelism = parallelism
+    var startHour = startHour
+}

.teamcity/components/settings.kt

@@ -0,0 +1,28 @@
+// specifies the default hour (UTC) at which tests should be triggered, if enabled
+var defaultStartHour = 0
+
+// specifies the default level of parallelism per-service-package
+var defaultParallelism = 20
+
+var locations = mapOf(
+        "public" to LocationConfiguration("westeurope", "eastus2", "francecentral", false),
+        "germany" to LocationConfiguration("germanynortheast", "germanycentral", "", false)
+)
+
+// specifies the list of Azure Environments where tests should be run nightly
+var runNightly = mapOf(
+        "public" to true
+)
+
+// specifies a list of services which should be run with a custom test configuration
+var serviceTestConfigurationOverrides = mapOf(
+        // The AKS API has a low rate limit
+        "containers" to testConfiguration(5, defaultStartHour),
+
+        // Data Lake has a low quota
+        "datalake" to testConfiguration(2, defaultStartHour),
+
+        // SignalR only allows provisioning one "Free" instance at a time,
+        // which is used in multiple tests
+        "signalr" to testConfiguration(1, defaultStartHour)
+)