From e34eb9739dbf6637bf32fdfa8c56b112f178d76b Mon Sep 17 00:00:00 2001 From: sweanan Date: Thu, 8 Jun 2023 13:54:46 -0500 Subject: [PATCH 01/10] synapse module for terratest --- .../terraform-azure-synapse-example/README.md | 32 +++++++ .../terraform-azure-synapse-example/main.tf | 87 +++++++++++++++++++ .../outputs.tf | 19 ++++ .../variables.tf | 67 ++++++++++++++ go.mod | 2 +- go.sum | 4 +- modules/azure/client_factory.go | 60 +++++++++++++ modules/azure/synapse.go | 86 ++++++++++++++++++ modules/azure/synapse_test.go | 35 ++++++++ .../terraform_azure_synapse_example_test.go | 59 +++++++++++++ 10 files changed, 448 insertions(+), 3 deletions(-) create mode 100644 examples/azure/terraform-azure-synapse-example/README.md create mode 100644 examples/azure/terraform-azure-synapse-example/main.tf create mode 100644 examples/azure/terraform-azure-synapse-example/outputs.tf create mode 100644 examples/azure/terraform-azure-synapse-example/variables.tf create mode 100644 modules/azure/synapse.go create mode 100644 modules/azure/synapse_test.go create mode 100644 test/azure/terraform_azure_synapse_example_test.go diff --git a/examples/azure/terraform-azure-synapse-example/README.md b/examples/azure/terraform-azure-synapse-example/README.md new file mode 100644 index 000000000..a6f7cc15f --- /dev/null +++ b/examples/azure/terraform-azure-synapse-example/README.md @@ -0,0 +1,32 @@ +# Terraform Azure Synapse Example + +This folder contains a Terraform module that deploys resources in [Azure](https://azure.microsoft.com/) to demonstrate how you can use Terratest to write automated tests for your Azure Terraform code. +This module deploys below resource: + +- A [Azure Synapse Analytics](https://azure.microsoft.com/en-us/products/synapse-analytics/). + +Check out [test/azure/terraform_azure_synapse_example_test.go](./../../../test/azure/terraform_azure_synapse_example_test.go) to see how you can write automated tests for this module and validate the configuration of the parameters and options. + +**WARNING**: This module and the automated tests for it deploy real resources into your Azure account which can cost you money. + +## Running this module manually + +1. Sign up for [Azure](https://azure.microsoft.com/). +1. Configure your Azure credentials using one of the [supported methods for Azure CLI + tools](https://docs.microsoft.com/en-us/cli/azure/azure-cli-configuration?view=azure-cli-latest) +1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH`. +1. Ensure [environment variables](../README.md#review-environment-variables) are available +1. Run `terraform init` +1. Run `terraform apply` +1. When you're done, run `terraform destroy`. + +## Running automated tests against this module + +1. Sign up for [Azure](https://azure.microsoft.com/) +1. Configure your Azure credentials using one of the [supported methods for Azure CLI + tools](https://docs.microsoft.com/en-us/cli/azure/azure-cli-configuration?view=azure-cli-latest) +1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH` +1. Configure your Terratest [Go test environment](../README.md) +1. `cd test/azure` +1. `go build terraform_azure_synapse_example_test.go` +1. `go test -v -timeout 60m -tags azure -run TestTerraformAzureSynapseExample` diff --git a/examples/azure/terraform-azure-synapse-example/main.tf b/examples/azure/terraform-azure-synapse-example/main.tf new file mode 100644 index 000000000..54cd2d462 --- /dev/null +++ b/examples/azure/terraform-azure-synapse-example/main.tf @@ -0,0 +1,87 @@ +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY AN AZURE Synapse Analytics +# This is an example of how to deploy an AZURE Synapse Analytics +# See test/terraform_azure_example_test.go for how to write automated tests for this code. +# --------------------------------------------------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------------------------------------------------- +# CONFIGURE OUR AZURE CONNECTION +# --------------------------------------------------------------------------------------------------------------------- + +provider "azurerm" { + version = "~>2.29.0" + features {} +} + +# --------------------------------------------------------------------------------------------------------------------- +# CREATE RANDOM PASSWORD +# --------------------------------------------------------------------------------------------------------------------- + +# Random password is used as an example to simplify the deployment and improve the security of the database. +# This is not as a production recommendation as the password is stored in the Terraform state file. +resource "random_password" "password" { + length = 16 + override_special = "-_%@" + min_upper = "1" + min_lower = "1" + min_numeric = "1" + min_special = "1" +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A RESOURCE GROUP +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_resource_group" "synapse_rg" { + name = "terratest-synapse-${var.postfix}" + location = var.location +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A STORAGE ACCOUNT +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_storage_account" "storage_account" { + name = "storage${var.postfix}" + resource_group_name = azurerm_resource_group.synapse_rg.name + location = azurerm_resource_group.synapse_rg.location + account_kind = var.storage_account_kind + account_tier = var.storage_account_tier + account_replication_type = var.storage_replication_type +} + + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A DATA LAKE GEN2 +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_storage_data_lake_gen2_filesystem" "dl_gen2" { + name = "dlgen2-${var.postfix}" + storage_account_id = azurerm_storage_account.storage_account.id +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A SYNAPSE WORKSPACE +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_synapse_workspace" "sy_workspace" { + name = "mysynapse${var.postfix}" + resource_group_name = azurerm_resource_group.synapse_rg.name + location = azurerm_resource_group.synapse_rg.location + storage_data_lake_gen2_filesystem_id = azurerm_storage_data_lake_gen2_filesystem.dl_gen2.id + sql_administrator_login = var.synapse_sql_user + sql_administrator_login_password = random_password.password.result + managed_virtual_network_enabled = true +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A SYNAPSE SQL POOL +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_synapse_sql_pool" "sy_pool" { + name = "sqlpool${var.postfix}" + synapse_workspace_id = azurerm_synapse_workspace.sy_workspace.id + sku_name = var.sy_sqlpool_sku_name + create_mode = var.sy_sqlpool_create_mode +} \ No newline at end of file diff --git a/examples/azure/terraform-azure-synapse-example/outputs.tf b/examples/azure/terraform-azure-synapse-example/outputs.tf new file mode 100644 index 000000000..d2212354a --- /dev/null +++ b/examples/azure/terraform-azure-synapse-example/outputs.tf @@ -0,0 +1,19 @@ +output "resource_group_name" { + value = azurerm_resource_group.synapse_rg.name +} + +output "synapse_storage_name" { + value = azurerm_storage_account.storage_account.name +} + +output "synapse_dlgen2_name" { + value = azurerm_storage_data_lake_gen2_filesystem.dl_gen2.name +} + +output "synapse_workspace_name" { + value = azurerm_synapse_workspace.sy_workspace.name +} + +output "synapse_sqlpool_name" { + value = azurerm_synapse_sql_pool.sy_pool.name +} diff --git a/examples/azure/terraform-azure-synapse-example/variables.tf b/examples/azure/terraform-azure-synapse-example/variables.tf new file mode 100644 index 000000000..507508c29 --- /dev/null +++ b/examples/azure/terraform-azure-synapse-example/variables.tf @@ -0,0 +1,67 @@ +# --------------------------------------------------------------------------------------------------------------------- +# ENVIRONMENT VARIABLES +# Define these secrets as environment variables +# --------------------------------------------------------------------------------------------------------------------- + +# ARM_CLIENT_ID +# ARM_CLIENT_SECRET +# ARM_SUBSCRIPTION_ID +# ARM_TENANT_ID + +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# You must provide a value for each of these parameters. +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# These parameters have reasonable defaults. +# --------------------------------------------------------------------------------------------------------------------- + +variable "location" { + description = "The supported azure location where the resource exists" + type = string + default = "West US2" +} + +variable "storage_account_kind" { + description = "The kind of storage account to set" + type = string + default = "StorageV2" +} + +variable "storage_account_tier" { + description = "The tier of storage account to set" + type = string + default = "Standard" +} + +variable "storage_replication_type" { + description = "The replication type of storage account to set" + type = string + default = "GRS" +} + +variable "synapse_sql_user" { + description = "The sql pool user password for synapse" + type = string + default = "sqladminuser" +} + +variable "sy_sqlpool_sku_name" { + description = "The sku name for the synapse sql pool" + type = string + default = "DW100c" +} + +variable "sy_sqlpool_create_mode" { + description = "The create mode for the synapse sql pool" + type = string + default = "Default" +} + +variable "postfix" { + description = "A postfix string to centrally mitigate resource name collisions." + type = string + default = "resource" +} \ No newline at end of file diff --git a/go.mod b/go.mod index 35b98bbb4..1459b1f01 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( cloud.google.com/go v0.105.0 // indirect cloud.google.com/go/storage v1.27.0 - github.com/Azure/azure-sdk-for-go v50.2.0+incompatible + github.com/Azure/azure-sdk-for-go v51.0.0+incompatible github.com/Azure/go-autorest/autorest v0.11.20 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect diff --git a/go.sum b/go.sum index e2a86b5ec..2b2b7c3bc 100644 --- a/go.sum +++ b/go.sum @@ -191,8 +191,8 @@ cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1V cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v50.2.0+incompatible h1:w9EF1btRfLLWbNEp6XvkMjeA6nQ3e1GZ2KNDqB/SjOQ= -github.com/Azure/azure-sdk-for-go v50.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v51.0.0+incompatible h1:p7blnyJSjJqf5jflHbSGhIhEpXIgIFmYZNg5uwqweso= +github.com/Azure/azure-sdk-for-go v51.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= diff --git a/modules/azure/client_factory.go b/modules/azure/client_factory.go index 383ff8980..d4dc53b51 100644 --- a/modules/azure/client_factory.go +++ b/modules/azure/client_factory.go @@ -28,6 +28,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-09-01/network" "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-06-01/subscriptions" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/Azure/azure-sdk-for-go/services/synapse/mgmt/2020-12-01/synapse" "github.com/Azure/azure-sdk-for-go/services/web/mgmt/2019-08-01/web" autorestAzure "github.com/Azure/go-autorest/autorest/azure" ) @@ -771,6 +772,65 @@ func CreateFrontDoorFrontendEndpointClientE(subscriptionID string) (*frontdoor.F return &client, nil } +// CreateSynapseWorkspaceClientE is a helper function that will setup a synapse client. +func CreateSynapseWorkspaceClientE(subscriptionID string) (*synapse.WorkspacesClient, error) { + // Validate Azure subscription ID + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + + // Lookup environment URI + baseURI, err := getBaseURI() + if err != nil { + return nil, err + } + + // Create a synapse client + synapseWorkspaceClient := synapse.NewWorkspacesClientWithBaseURI(baseURI, subscriptionID) + + // synapse.SQLPoolsClient + // Create an authorizer + authorizer, err := NewAuthorizer() + if err != nil { + return nil, err + } + + // Attach authorizer to the client + synapseWorkspaceClient.Authorizer = *authorizer + + return &synapseWorkspaceClient, nil +} + +// CreateSynapseSqlPoolClientE is a helper function that will setup a synapse client. +func CreateSynapseSqlPoolClientE(subscriptionID string) (*synapse.SQLPoolsClient, error) { + // Validate Azure subscription ID + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + + // Lookup environment URI + baseURI, err := getBaseURI() + if err != nil { + return nil, err + } + + // Create a synapse client + synapseSqlPoolClient := synapse.NewSQLPoolsClientWithBaseURI(baseURI, subscriptionID) + + // Create an authorizer + authorizer, err := NewAuthorizer() + if err != nil { + return nil, err + } + + // Attach authorizer to the client + synapseSqlPoolClient.Authorizer = *authorizer + + return &synapseSqlPoolClient, nil +} + // GetKeyVaultURISuffixE returns the proper KeyVault URI suffix for the configured Azure environment. // This function would fail the test if there is an error. func GetKeyVaultURISuffixE() (string, error) { diff --git a/modules/azure/synapse.go b/modules/azure/synapse.go new file mode 100644 index 000000000..f40f22351 --- /dev/null +++ b/modules/azure/synapse.go @@ -0,0 +1,86 @@ +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/synapse/mgmt/2020-12-01/synapse" + "github.com/gruntwork-io/terratest/modules/testing" + "github.com/stretchr/testify/require" +) + +// GetSynapseWorkspaceClientE is a helper function that will setup a mysql server client. +func GetSynapseWorkspaceClientE(subscriptionID string) (*synapse.WorkspacesClient, error) { + // Validate Azure subscription ID + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + + // Create a synapse client + synapseClient := synapse.NewWorkspacesClient(subscriptionID) + + // Create an authorizer + authorizer, err := NewAuthorizer() + if err != nil { + return nil, err + } + + // Attach authorizer to the client + synapseClient.Authorizer = *authorizer + + return &synapseClient, nil +} + +// GetSynapseWorkspace is a helper function that gets the synapse workspace. +// This function would fail the test if there is an error. +func GetSynapseWorkspace(t testing.TestingT, resGroupName string, workspaceName string, subscriptionID string) *synapse.Workspace { + Workspace, err := GetSynapseWorkspaceE(t, subscriptionID, resGroupName, workspaceName) + require.NoError(t, err) + + return Workspace +} + +// GetSynapseSqlPool is a helper function that gets the synapse workspace. +// This function would fail the test if there is an error. +func GetSynapseSqlPool(t testing.TestingT, resGroupName string, workspaceName string, sqlPoolName string, subscriptionID string) *synapse.SQLPool { + SQLPool, err := GetSynapseSqlPoolE(t, subscriptionID, resGroupName, workspaceName, sqlPoolName) + require.NoError(t, err) + + return SQLPool +} + +// GetSynapseWorkspaceE is a helper function that gets the workspace. +func GetSynapseWorkspaceE(t testing.TestingT, subscriptionID string, resGroupName string, workspaceName string) (*synapse.Workspace, error) { + // Create a synapse client + synapseClient, err := CreateSynapseWorkspaceClientE(subscriptionID) + if err != nil { + return nil, err + } + + // Get the corresponding synapse workspace + synapseWorkspace, err := synapseClient.Get(context.Background(), resGroupName, workspaceName) + if err != nil { + return nil, err + } + + //Return synapse workspace + return &synapseWorkspace, nil +} + +// GetSynapseSqlPoolE is a helper function that gets the synapse sql pool. +func GetSynapseSqlPoolE(t testing.TestingT, subscriptionID string, resGroupName string, workspaceName string, sqlPoolName string) (*synapse.SQLPool, error) { + // Create a synapse client + synapseSqlPoolClient, err := CreateSynapseSqlPoolClientE(subscriptionID) + if err != nil { + return nil, err + } + + // Get the corresponding synapse workspace + synapseSqlPool, err := synapseSqlPoolClient.Get(context.Background(), resGroupName, workspaceName, sqlPoolName) + if err != nil { + return nil, err + } + + //Return synapse workspace + return &synapseSqlPool, nil +} diff --git a/modules/azure/synapse_test.go b/modules/azure/synapse_test.go new file mode 100644 index 000000000..672530675 --- /dev/null +++ b/modules/azure/synapse_test.go @@ -0,0 +1,35 @@ +package azure + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +/* +The below tests are currently stubbed out, with the expectation that they will throw errors. +If/when CRUD methods are introduced for Azure Synapse, these tests can be extended +*/ + +func TestGetSynapseWorkspaceE(t *testing.T) { + t.Parallel() + + resGroupName := "" + subscriptionID := "" + workspaceName := "" + + _, err := GetSynapseWorkspaceE(t, subscriptionID, resGroupName, workspaceName) + require.Error(t, err) +} + +func TestGetSynapseSqlPoolE(t *testing.T) { + t.Parallel() + + resGroupName := "" + subscriptionID := "" + workspaceName := "" + sqlPoolName := "" + + _, err := GetSynapseSqlPoolE(t, subscriptionID, resGroupName, workspaceName, sqlPoolName) + require.Error(t, err) +} diff --git a/test/azure/terraform_azure_synapse_example_test.go b/test/azure/terraform_azure_synapse_example_test.go new file mode 100644 index 000000000..f610a2802 --- /dev/null +++ b/test/azure/terraform_azure_synapse_example_test.go @@ -0,0 +1,59 @@ +package test + +import ( + "strings" + "testing" + + "github.com/gruntwork-io/terratest/modules/azure" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" +) + +func TestTerraformAzureSynapseExample(t *testing.T) { + t.Parallel() + + uniquePostfix := strings.ToLower(random.UniqueId()) + expectedSynapseSqlUser := "sqladminuser" + expectedSynapseProvisioningState := "Succeeded" + expectedLocation := "West US2" + expectedSyPoolSkuName := "DW100c" + + // website::tag::1:: Configure Terraform setting up a path to Terraform code. + terraformOptions := &terraform.Options{ + // The path to where our Terraform code is located + TerraformDir: "../../examples/azure/terraform-azure-synapse-example", + Vars: map[string]interface{}{ + "postfix": uniquePostfix, + "synapse_sql_user": expectedSynapseSqlUser, + "location": expectedLocation, + "sy_sqlpool_sku_name": expectedSyPoolSkuName, + }, + } + + // website::tag::4:: At the end of the test, run `terraform destroy` to clean up any resources that were created + defer terraform.Destroy(t, terraformOptions) + + // website::tag::2:: Run `terraform init` and `terraform apply`. Fail the test if there are any errors. + terraform.InitAndApply(t, terraformOptions) + // terraform.InitE(t, terraformOptions) + + // website::tag::3:: Run `terraform output` to get the values of output variables + expectedResourceGroupName := terraform.Output(t, terraformOptions, "resource_group_name") + expectedSyDLgen2Name := terraform.Output(t, terraformOptions, "synapse_dlgen2_name") + expectedSyWorkspaceName := terraform.Output(t, terraformOptions, "synapse_workspace_name") + expectedSqlPoolName := terraform.Output(t, terraformOptions, "synapse_sqlpool_name") + + // website::tag::4:: Get synapse details and assert them against the terraform output + actualSynapseWorkspace := azure.GetSynapseWorkspace(t, expectedResourceGroupName, expectedSyWorkspaceName, "") + actualSynapseSqlPool := azure.GetSynapseSqlPool(t, expectedResourceGroupName, expectedSyWorkspaceName, expectedSqlPoolName, "") + + assert.Equal(t, expectedSyWorkspaceName, *actualSynapseWorkspace.Name) + assert.Equal(t, expectedSynapseSqlUser, *actualSynapseWorkspace.WorkspaceProperties.SQLAdministratorLogin) + assert.Equal(t, expectedSynapseProvisioningState, *actualSynapseWorkspace.WorkspaceProperties.ProvisioningState) + assert.Equal(t, expectedLocation, *actualSynapseWorkspace.Location) + assert.Equal(t, expectedSyDLgen2Name, *actualSynapseWorkspace.WorkspaceProperties.DefaultDataLakeStorage.Filesystem) + + assert.Equal(t, expectedSqlPoolName, *actualSynapseSqlPool.Name) + assert.Equal(t, expectedSyPoolSkuName, *actualSynapseSqlPool.Sku.Name) +} From 2236fbcfcbfc9dcb3957bd6e12d3d0eabb8edc19 Mon Sep 17 00:00:00 2001 From: sweanan Date: Thu, 8 Jun 2023 15:26:59 -0500 Subject: [PATCH 02/10] Upadted based on PR comments --- .../terraform-azure-synapse-example/main.tf | 10 ++++---- .../outputs.tf | 4 ++-- .../variables.tf | 6 ++--- modules/azure/client_factory.go | 1 - modules/azure/synapse.go | 23 ------------------- .../terraform_azure_synapse_example_test.go | 8 +++---- 6 files changed, 14 insertions(+), 38 deletions(-) diff --git a/examples/azure/terraform-azure-synapse-example/main.tf b/examples/azure/terraform-azure-synapse-example/main.tf index 54cd2d462..9223ba001 100644 --- a/examples/azure/terraform-azure-synapse-example/main.tf +++ b/examples/azure/terraform-azure-synapse-example/main.tf @@ -48,7 +48,7 @@ resource "azurerm_storage_account" "storage_account" { location = azurerm_resource_group.synapse_rg.location account_kind = var.storage_account_kind account_tier = var.storage_account_tier - account_replication_type = var.storage_replication_type + account_replication_type = var.storage_account_replication_type } @@ -65,7 +65,7 @@ resource "azurerm_storage_data_lake_gen2_filesystem" "dl_gen2" { # DEPLOY A SYNAPSE WORKSPACE # --------------------------------------------------------------------------------------------------------------------- -resource "azurerm_synapse_workspace" "sy_workspace" { +resource "azurerm_synapse_workspace" "synapse_workspace" { name = "mysynapse${var.postfix}" resource_group_name = azurerm_resource_group.synapse_rg.name location = azurerm_resource_group.synapse_rg.location @@ -79,9 +79,9 @@ resource "azurerm_synapse_workspace" "sy_workspace" { # DEPLOY A SYNAPSE SQL POOL # --------------------------------------------------------------------------------------------------------------------- -resource "azurerm_synapse_sql_pool" "sy_pool" { +resource "azurerm_synapse_sql_pool" "synapse_pool" { name = "sqlpool${var.postfix}" synapse_workspace_id = azurerm_synapse_workspace.sy_workspace.id - sku_name = var.sy_sqlpool_sku_name - create_mode = var.sy_sqlpool_create_mode + sku_name = var.synapse_sqlpool_sku_name + create_mode = var.synapse_sqlpool_create_mode } \ No newline at end of file diff --git a/examples/azure/terraform-azure-synapse-example/outputs.tf b/examples/azure/terraform-azure-synapse-example/outputs.tf index d2212354a..68fede646 100644 --- a/examples/azure/terraform-azure-synapse-example/outputs.tf +++ b/examples/azure/terraform-azure-synapse-example/outputs.tf @@ -11,9 +11,9 @@ output "synapse_dlgen2_name" { } output "synapse_workspace_name" { - value = azurerm_synapse_workspace.sy_workspace.name + value = azurerm_synapse_workspace.synapse_workspace.name } output "synapse_sqlpool_name" { - value = azurerm_synapse_sql_pool.sy_pool.name + value = azurerm_synapse_sql_pool.synapse_pool.name } diff --git a/examples/azure/terraform-azure-synapse-example/variables.tf b/examples/azure/terraform-azure-synapse-example/variables.tf index 507508c29..33cb9cb13 100644 --- a/examples/azure/terraform-azure-synapse-example/variables.tf +++ b/examples/azure/terraform-azure-synapse-example/variables.tf @@ -36,7 +36,7 @@ variable "storage_account_tier" { default = "Standard" } -variable "storage_replication_type" { +variable "storage_account_replication_type" { description = "The replication type of storage account to set" type = string default = "GRS" @@ -48,13 +48,13 @@ variable "synapse_sql_user" { default = "sqladminuser" } -variable "sy_sqlpool_sku_name" { +variable "synapse_sqlpool_sku_name" { description = "The sku name for the synapse sql pool" type = string default = "DW100c" } -variable "sy_sqlpool_create_mode" { +variable "synapse_sqlpool_create_mode" { description = "The create mode for the synapse sql pool" type = string default = "Default" diff --git a/modules/azure/client_factory.go b/modules/azure/client_factory.go index d4dc53b51..c1e891fd7 100644 --- a/modules/azure/client_factory.go +++ b/modules/azure/client_factory.go @@ -789,7 +789,6 @@ func CreateSynapseWorkspaceClientE(subscriptionID string) (*synapse.WorkspacesCl // Create a synapse client synapseWorkspaceClient := synapse.NewWorkspacesClientWithBaseURI(baseURI, subscriptionID) - // synapse.SQLPoolsClient // Create an authorizer authorizer, err := NewAuthorizer() if err != nil { diff --git a/modules/azure/synapse.go b/modules/azure/synapse.go index f40f22351..079c1cec9 100644 --- a/modules/azure/synapse.go +++ b/modules/azure/synapse.go @@ -8,29 +8,6 @@ import ( "github.com/stretchr/testify/require" ) -// GetSynapseWorkspaceClientE is a helper function that will setup a mysql server client. -func GetSynapseWorkspaceClientE(subscriptionID string) (*synapse.WorkspacesClient, error) { - // Validate Azure subscription ID - subscriptionID, err := getTargetAzureSubscription(subscriptionID) - if err != nil { - return nil, err - } - - // Create a synapse client - synapseClient := synapse.NewWorkspacesClient(subscriptionID) - - // Create an authorizer - authorizer, err := NewAuthorizer() - if err != nil { - return nil, err - } - - // Attach authorizer to the client - synapseClient.Authorizer = *authorizer - - return &synapseClient, nil -} - // GetSynapseWorkspace is a helper function that gets the synapse workspace. // This function would fail the test if there is an error. func GetSynapseWorkspace(t testing.TestingT, resGroupName string, workspaceName string, subscriptionID string) *synapse.Workspace { diff --git a/test/azure/terraform_azure_synapse_example_test.go b/test/azure/terraform_azure_synapse_example_test.go index f610a2802..468706761 100644 --- a/test/azure/terraform_azure_synapse_example_test.go +++ b/test/azure/terraform_azure_synapse_example_test.go @@ -24,10 +24,10 @@ func TestTerraformAzureSynapseExample(t *testing.T) { // The path to where our Terraform code is located TerraformDir: "../../examples/azure/terraform-azure-synapse-example", Vars: map[string]interface{}{ - "postfix": uniquePostfix, - "synapse_sql_user": expectedSynapseSqlUser, - "location": expectedLocation, - "sy_sqlpool_sku_name": expectedSyPoolSkuName, + "postfix": uniquePostfix, + "synapse_sql_user": expectedSynapseSqlUser, + "location": expectedLocation, + "synapse_sqlpool_sku_name": expectedSyPoolSkuName, }, } From 10903eb44530abd135a416ba42c2daf5c368c125 Mon Sep 17 00:00:00 2001 From: sweanan Date: Fri, 9 Jun 2023 07:05:43 -0500 Subject: [PATCH 03/10] update typo --- examples/azure/terraform-azure-synapse-example/main.tf | 2 +- go.mod | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/azure/terraform-azure-synapse-example/main.tf b/examples/azure/terraform-azure-synapse-example/main.tf index 9223ba001..f85cfbc47 100644 --- a/examples/azure/terraform-azure-synapse-example/main.tf +++ b/examples/azure/terraform-azure-synapse-example/main.tf @@ -81,7 +81,7 @@ resource "azurerm_synapse_workspace" "synapse_workspace" { resource "azurerm_synapse_sql_pool" "synapse_pool" { name = "sqlpool${var.postfix}" - synapse_workspace_id = azurerm_synapse_workspace.sy_workspace.id + synapse_workspace_id = azurerm_synapse_workspace.synapse_workspace.id sku_name = var.synapse_sqlpool_sku_name create_mode = var.synapse_sqlpool_create_mode } \ No newline at end of file diff --git a/go.mod b/go.mod index 1459b1f01..6c3ba29d5 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( ) require ( + cloud.google.com/go v0.105.0 // indirect cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.7.0 // indirect From a97b635771cfa3dc5f21d237451fcee425ebfc8c Mon Sep 17 00:00:00 2001 From: sweanan Date: Mon, 12 Jun 2023 07:38:14 -0500 Subject: [PATCH 04/10] update the format --- examples/azure/terraform-azure-synapse-example/main.tf | 4 ++-- examples/azure/terraform-azure-synapse-example/outputs.tf | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/azure/terraform-azure-synapse-example/main.tf b/examples/azure/terraform-azure-synapse-example/main.tf index f85cfbc47..8a66da1b0 100644 --- a/examples/azure/terraform-azure-synapse-example/main.tf +++ b/examples/azure/terraform-azure-synapse-example/main.tf @@ -10,8 +10,8 @@ # --------------------------------------------------------------------------------------------------------------------- provider "azurerm" { - version = "~>2.29.0" - features {} + version = "~>2.29.0" + features {} } # --------------------------------------------------------------------------------------------------------------------- diff --git a/examples/azure/terraform-azure-synapse-example/outputs.tf b/examples/azure/terraform-azure-synapse-example/outputs.tf index 68fede646..b7dd6a601 100644 --- a/examples/azure/terraform-azure-synapse-example/outputs.tf +++ b/examples/azure/terraform-azure-synapse-example/outputs.tf @@ -1,6 +1,6 @@ output "resource_group_name" { value = azurerm_resource_group.synapse_rg.name -} +} output "synapse_storage_name" { value = azurerm_storage_account.storage_account.name @@ -15,5 +15,5 @@ output "synapse_workspace_name" { } output "synapse_sqlpool_name" { - value = azurerm_synapse_sql_pool.synapse_pool.name + value = azurerm_synapse_sql_pool.synapse_pool.name } From 4f082da289c2d3c7251a9fdc60703e7c1a16d6a4 Mon Sep 17 00:00:00 2001 From: sweanan Date: Mon, 12 Jun 2023 08:59:14 -0500 Subject: [PATCH 05/10] added build tags --- go.mod | 1 - test/azure/terraform_azure_synapse_example_test.go | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6c3ba29d5..1459b1f01 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,6 @@ require ( ) require ( - cloud.google.com/go v0.105.0 // indirect cloud.google.com/go/compute v1.12.1 // indirect cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.7.0 // indirect diff --git a/test/azure/terraform_azure_synapse_example_test.go b/test/azure/terraform_azure_synapse_example_test.go index 468706761..c779115b6 100644 --- a/test/azure/terraform_azure_synapse_example_test.go +++ b/test/azure/terraform_azure_synapse_example_test.go @@ -1,3 +1,9 @@ +//go:build azure +// +build azure + +// NOTE: We use build tags to differentiate azure testing because we currently do not have azure access setup for +// CircleCI. + package test import ( From e94bb8165b321d61dbfb9e9cc18400df4fc4435f Mon Sep 17 00:00:00 2001 From: sweanan Date: Mon, 12 Jun 2023 15:53:33 -0500 Subject: [PATCH 06/10] update the version --- examples/azure/terraform-azure-synapse-example/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/azure/terraform-azure-synapse-example/main.tf b/examples/azure/terraform-azure-synapse-example/main.tf index 8a66da1b0..ab4f31d99 100644 --- a/examples/azure/terraform-azure-synapse-example/main.tf +++ b/examples/azure/terraform-azure-synapse-example/main.tf @@ -10,7 +10,7 @@ # --------------------------------------------------------------------------------------------------------------------- provider "azurerm" { - version = "~>2.29.0" + version = "~>2.93.0" features {} } From 9e432b944d614c92d91937790d3b90a5f0fc615f Mon Sep 17 00:00:00 2001 From: Hadwa Gaber Date: Mon, 12 Jun 2023 16:13:50 -0700 Subject: [PATCH 07/10] remove logger calls --- modules/azure/region.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/azure/region.go b/modules/azure/region.go index e36733531..c478beeb2 100644 --- a/modules/azure/region.go +++ b/modules/azure/region.go @@ -4,7 +4,6 @@ import ( "context" "github.com/gruntwork-io/terratest/modules/collections" - "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/testing" ) @@ -115,7 +114,6 @@ func GetRandomRegionE(t testing.TestingT, approvedRegions []string, forbiddenReg regionsToPickFrom = collections.ListSubtract(regionsToPickFrom, forbiddenRegions) region := random.RandomString(regionsToPickFrom) - logger.Logf(t, "Using region %s", region) return region, nil } @@ -138,7 +136,6 @@ func GetAllAzureRegions(t testing.TestingT, subscriptionID string) []string { // GetAllAzureRegionsE gets the list of Azure regions available in this subscription func GetAllAzureRegionsE(t testing.TestingT, subscriptionID string) ([]string, error) { - logger.Log(t, "Looking up all Azure regions available in this account") // Validate Azure subscription ID subscriptionID, err := getTargetAzureSubscription(subscriptionID) From 2c863d6e13cc43ec23ab3413b96bb74927b34c85 Mon Sep 17 00:00:00 2001 From: Hadwa Gaber Date: Mon, 12 Jun 2023 16:21:30 -0700 Subject: [PATCH 08/10] re-add logger calls --- modules/azure/region.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/azure/region.go b/modules/azure/region.go index c478beeb2..e36733531 100644 --- a/modules/azure/region.go +++ b/modules/azure/region.go @@ -4,6 +4,7 @@ import ( "context" "github.com/gruntwork-io/terratest/modules/collections" + "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/testing" ) @@ -114,6 +115,7 @@ func GetRandomRegionE(t testing.TestingT, approvedRegions []string, forbiddenReg regionsToPickFrom = collections.ListSubtract(regionsToPickFrom, forbiddenRegions) region := random.RandomString(regionsToPickFrom) + logger.Logf(t, "Using region %s", region) return region, nil } @@ -136,6 +138,7 @@ func GetAllAzureRegions(t testing.TestingT, subscriptionID string) []string { // GetAllAzureRegionsE gets the list of Azure regions available in this subscription func GetAllAzureRegionsE(t testing.TestingT, subscriptionID string) ([]string, error) { + logger.Log(t, "Looking up all Azure regions available in this account") // Validate Azure subscription ID subscriptionID, err := getTargetAzureSubscription(subscriptionID) From 2623f81295ec3ac4a292a27435f2112e3c2d7876 Mon Sep 17 00:00:00 2001 From: sweanan Date: Mon, 12 Jun 2023 17:15:46 -0500 Subject: [PATCH 09/10] update the ci yml to fix linter issue --- .github/workflows/ci.yml | 2 +- .../azure/terraform-azure-aks-example/output.tf | 15 ++++++++++----- modules/azure/region.go | 6 +++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 670798899..03813b91b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: ref: ${{ github.event.inputs.branch }} - name: install golangci-lint binary run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v1.31.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v1.53.2 - name: lint modules/azure folder id: azure_module_lint run: | diff --git a/examples/azure/terraform-azure-aks-example/output.tf b/examples/azure/terraform-azure-aks-example/output.tf index a06b6ca89..2bd5b736e 100644 --- a/examples/azure/terraform-azure-aks-example/output.tf +++ b/examples/azure/terraform-azure-aks-example/output.tf @@ -3,19 +3,23 @@ output "client_key" { } output "client_certificate" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate + value = azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate + sensitive = true } output "cluster_ca_certificate" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate + value = azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate + sensitive = true } output "cluster_username" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.username + value = azurerm_kubernetes_cluster.k8s.kube_config.0.username + sensitive = true } output "cluster_password" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.password + value = azurerm_kubernetes_cluster.k8s.kube_config.0.password + sensitive = true } output "kube_config" { @@ -24,5 +28,6 @@ output "kube_config" { } output "host" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.host + value = azurerm_kubernetes_cluster.k8s.kube_config.0.host + sensitive = true } \ No newline at end of file diff --git a/modules/azure/region.go b/modules/azure/region.go index e36733531..7d6c57a60 100644 --- a/modules/azure/region.go +++ b/modules/azure/region.go @@ -4,7 +4,7 @@ import ( "context" "github.com/gruntwork-io/terratest/modules/collections" - "github.com/gruntwork-io/terratest/modules/logger" + // "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/testing" ) @@ -115,7 +115,7 @@ func GetRandomRegionE(t testing.TestingT, approvedRegions []string, forbiddenReg regionsToPickFrom = collections.ListSubtract(regionsToPickFrom, forbiddenRegions) region := random.RandomString(regionsToPickFrom) - logger.Logf(t, "Using region %s", region) + // logger.Logf(t, "Using region %s", region) return region, nil } @@ -138,7 +138,7 @@ func GetAllAzureRegions(t testing.TestingT, subscriptionID string) []string { // GetAllAzureRegionsE gets the list of Azure regions available in this subscription func GetAllAzureRegionsE(t testing.TestingT, subscriptionID string) ([]string, error) { - logger.Log(t, "Looking up all Azure regions available in this account") + // logger.Log(t, "Looking up all Azure regions available in this account") // Validate Azure subscription ID subscriptionID, err := getTargetAzureSubscription(subscriptionID) From 2e01363b439ef3040189eb5de2aa50b4b4aa908c Mon Sep 17 00:00:00 2001 From: sweanan Date: Mon, 12 Jun 2023 19:13:15 -0500 Subject: [PATCH 10/10] revert back the ci yaml changes --- .github/workflows/ci.yml | 2 +- .../azure/terraform-azure-aks-example/output.tf | 15 +++++---------- modules/azure/region.go | 6 +++--- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03813b91b..670798899 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: ref: ${{ github.event.inputs.branch }} - name: install golangci-lint binary run: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v1.53.2 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v1.31.0 - name: lint modules/azure folder id: azure_module_lint run: | diff --git a/examples/azure/terraform-azure-aks-example/output.tf b/examples/azure/terraform-azure-aks-example/output.tf index 2bd5b736e..a06b6ca89 100644 --- a/examples/azure/terraform-azure-aks-example/output.tf +++ b/examples/azure/terraform-azure-aks-example/output.tf @@ -3,23 +3,19 @@ output "client_key" { } output "client_certificate" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate - sensitive = true + value = azurerm_kubernetes_cluster.k8s.kube_config.0.client_certificate } output "cluster_ca_certificate" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate - sensitive = true + value = azurerm_kubernetes_cluster.k8s.kube_config.0.cluster_ca_certificate } output "cluster_username" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.username - sensitive = true + value = azurerm_kubernetes_cluster.k8s.kube_config.0.username } output "cluster_password" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.password - sensitive = true + value = azurerm_kubernetes_cluster.k8s.kube_config.0.password } output "kube_config" { @@ -28,6 +24,5 @@ output "kube_config" { } output "host" { - value = azurerm_kubernetes_cluster.k8s.kube_config.0.host - sensitive = true + value = azurerm_kubernetes_cluster.k8s.kube_config.0.host } \ No newline at end of file diff --git a/modules/azure/region.go b/modules/azure/region.go index 7d6c57a60..e36733531 100644 --- a/modules/azure/region.go +++ b/modules/azure/region.go @@ -4,7 +4,7 @@ import ( "context" "github.com/gruntwork-io/terratest/modules/collections" - // "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/gruntwork-io/terratest/modules/testing" ) @@ -115,7 +115,7 @@ func GetRandomRegionE(t testing.TestingT, approvedRegions []string, forbiddenReg regionsToPickFrom = collections.ListSubtract(regionsToPickFrom, forbiddenRegions) region := random.RandomString(regionsToPickFrom) - // logger.Logf(t, "Using region %s", region) + logger.Logf(t, "Using region %s", region) return region, nil } @@ -138,7 +138,7 @@ func GetAllAzureRegions(t testing.TestingT, subscriptionID string) []string { // GetAllAzureRegionsE gets the list of Azure regions available in this subscription func GetAllAzureRegionsE(t testing.TestingT, subscriptionID string) ([]string, error) { - // logger.Log(t, "Looking up all Azure regions available in this account") + logger.Log(t, "Looking up all Azure regions available in this account") // Validate Azure subscription ID subscriptionID, err := getTargetAzureSubscription(subscriptionID)