diff --git a/examples/azure/terraform-azure-synapse-example/README.md b/examples/azure/terraform-azure-synapse-example/README.md new file mode 100644 index 000000000..a6f7cc15f --- /dev/null +++ b/examples/azure/terraform-azure-synapse-example/README.md @@ -0,0 +1,32 @@ +# Terraform Azure Synapse Example + +This folder contains a Terraform module that deploys resources in [Azure](https://azure.microsoft.com/) to demonstrate how you can use Terratest to write automated tests for your Azure Terraform code. +This module deploys below resource: + +- A [Azure Synapse Analytics](https://azure.microsoft.com/en-us/products/synapse-analytics/). + +Check out [test/azure/terraform_azure_synapse_example_test.go](./../../../test/azure/terraform_azure_synapse_example_test.go) to see how you can write automated tests for this module and validate the configuration of the parameters and options. + +**WARNING**: This module and the automated tests for it deploy real resources into your Azure account which can cost you money. + +## Running this module manually + +1. Sign up for [Azure](https://azure.microsoft.com/). +1. Configure your Azure credentials using one of the [supported methods for Azure CLI + tools](https://docs.microsoft.com/en-us/cli/azure/azure-cli-configuration?view=azure-cli-latest) +1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH`. +1. Ensure [environment variables](../README.md#review-environment-variables) are available +1. Run `terraform init` +1. Run `terraform apply` +1. When you're done, run `terraform destroy`. + +## Running automated tests against this module + +1. Sign up for [Azure](https://azure.microsoft.com/) +1. Configure your Azure credentials using one of the [supported methods for Azure CLI + tools](https://docs.microsoft.com/en-us/cli/azure/azure-cli-configuration?view=azure-cli-latest) +1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH` +1. Configure your Terratest [Go test environment](../README.md) +1. `cd test/azure` +1. `go build terraform_azure_synapse_example_test.go` +1. `go test -v -timeout 60m -tags azure -run TestTerraformAzureSynapseExample` diff --git a/examples/azure/terraform-azure-synapse-example/main.tf b/examples/azure/terraform-azure-synapse-example/main.tf new file mode 100644 index 000000000..ab4f31d99 --- /dev/null +++ b/examples/azure/terraform-azure-synapse-example/main.tf @@ -0,0 +1,87 @@ +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY AN AZURE Synapse Analytics +# This is an example of how to deploy an AZURE Synapse Analytics +# See test/terraform_azure_example_test.go for how to write automated tests for this code. +# --------------------------------------------------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------------------------------------------------- +# CONFIGURE OUR AZURE CONNECTION +# --------------------------------------------------------------------------------------------------------------------- + +provider "azurerm" { + version = "~>2.93.0" + features {} +} + +# --------------------------------------------------------------------------------------------------------------------- +# CREATE RANDOM PASSWORD +# --------------------------------------------------------------------------------------------------------------------- + +# Random password is used as an example to simplify the deployment and improve the security of the database. +# This is not as a production recommendation as the password is stored in the Terraform state file. +resource "random_password" "password" { + length = 16 + override_special = "-_%@" + min_upper = "1" + min_lower = "1" + min_numeric = "1" + min_special = "1" +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A RESOURCE GROUP +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_resource_group" "synapse_rg" { + name = "terratest-synapse-${var.postfix}" + location = var.location +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A STORAGE ACCOUNT +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_storage_account" "storage_account" { + name = "storage${var.postfix}" + resource_group_name = azurerm_resource_group.synapse_rg.name + location = azurerm_resource_group.synapse_rg.location + account_kind = var.storage_account_kind + account_tier = var.storage_account_tier + account_replication_type = var.storage_account_replication_type +} + + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A DATA LAKE GEN2 +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_storage_data_lake_gen2_filesystem" "dl_gen2" { + name = "dlgen2-${var.postfix}" + storage_account_id = azurerm_storage_account.storage_account.id +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A SYNAPSE WORKSPACE +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_synapse_workspace" "synapse_workspace" { + name = "mysynapse${var.postfix}" + resource_group_name = azurerm_resource_group.synapse_rg.name + location = azurerm_resource_group.synapse_rg.location + storage_data_lake_gen2_filesystem_id = azurerm_storage_data_lake_gen2_filesystem.dl_gen2.id + sql_administrator_login = var.synapse_sql_user + sql_administrator_login_password = random_password.password.result + managed_virtual_network_enabled = true +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A SYNAPSE SQL POOL +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_synapse_sql_pool" "synapse_pool" { + name = "sqlpool${var.postfix}" + synapse_workspace_id = azurerm_synapse_workspace.synapse_workspace.id + sku_name = var.synapse_sqlpool_sku_name + create_mode = var.synapse_sqlpool_create_mode +} \ No newline at end of file diff --git a/examples/azure/terraform-azure-synapse-example/outputs.tf b/examples/azure/terraform-azure-synapse-example/outputs.tf new file mode 100644 index 000000000..b7dd6a601 --- /dev/null +++ b/examples/azure/terraform-azure-synapse-example/outputs.tf @@ -0,0 +1,19 @@ +output "resource_group_name" { + value = azurerm_resource_group.synapse_rg.name +} + +output "synapse_storage_name" { + value = azurerm_storage_account.storage_account.name +} + +output "synapse_dlgen2_name" { + value = azurerm_storage_data_lake_gen2_filesystem.dl_gen2.name +} + +output "synapse_workspace_name" { + value = azurerm_synapse_workspace.synapse_workspace.name +} + +output "synapse_sqlpool_name" { + value = azurerm_synapse_sql_pool.synapse_pool.name +} diff --git a/examples/azure/terraform-azure-synapse-example/variables.tf b/examples/azure/terraform-azure-synapse-example/variables.tf new file mode 100644 index 000000000..33cb9cb13 --- /dev/null +++ b/examples/azure/terraform-azure-synapse-example/variables.tf @@ -0,0 +1,67 @@ +# --------------------------------------------------------------------------------------------------------------------- +# ENVIRONMENT VARIABLES +# Define these secrets as environment variables +# --------------------------------------------------------------------------------------------------------------------- + +# ARM_CLIENT_ID +# ARM_CLIENT_SECRET +# ARM_SUBSCRIPTION_ID +# ARM_TENANT_ID + +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# You must provide a value for each of these parameters. +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# These parameters have reasonable defaults. +# --------------------------------------------------------------------------------------------------------------------- + +variable "location" { + description = "The supported azure location where the resource exists" + type = string + default = "West US2" +} + +variable "storage_account_kind" { + description = "The kind of storage account to set" + type = string + default = "StorageV2" +} + +variable "storage_account_tier" { + description = "The tier of storage account to set" + type = string + default = "Standard" +} + +variable "storage_account_replication_type" { + description = "The replication type of storage account to set" + type = string + default = "GRS" +} + +variable "synapse_sql_user" { + description = "The sql pool user password for synapse" + type = string + default = "sqladminuser" +} + +variable "synapse_sqlpool_sku_name" { + description = "The sku name for the synapse sql pool" + type = string + default = "DW100c" +} + +variable "synapse_sqlpool_create_mode" { + description = "The create mode for the synapse sql pool" + type = string + default = "Default" +} + +variable "postfix" { + description = "A postfix string to centrally mitigate resource name collisions." + type = string + default = "resource" +} \ No newline at end of file diff --git a/go.mod b/go.mod index 35b98bbb4..1459b1f01 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.19 require ( cloud.google.com/go v0.105.0 // indirect cloud.google.com/go/storage v1.27.0 - github.com/Azure/azure-sdk-for-go v50.2.0+incompatible + github.com/Azure/azure-sdk-for-go v51.0.0+incompatible github.com/Azure/go-autorest/autorest v0.11.20 github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect diff --git a/go.sum b/go.sum index e2a86b5ec..2b2b7c3bc 100644 --- a/go.sum +++ b/go.sum @@ -191,8 +191,8 @@ cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1V cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v50.2.0+incompatible h1:w9EF1btRfLLWbNEp6XvkMjeA6nQ3e1GZ2KNDqB/SjOQ= -github.com/Azure/azure-sdk-for-go v50.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v51.0.0+incompatible h1:p7blnyJSjJqf5jflHbSGhIhEpXIgIFmYZNg5uwqweso= +github.com/Azure/azure-sdk-for-go v51.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= diff --git a/modules/azure/client_factory.go b/modules/azure/client_factory.go index 383ff8980..c1e891fd7 100644 --- a/modules/azure/client_factory.go +++ b/modules/azure/client_factory.go @@ -28,6 +28,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-09-01/network" "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-06-01/subscriptions" "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/Azure/azure-sdk-for-go/services/synapse/mgmt/2020-12-01/synapse" "github.com/Azure/azure-sdk-for-go/services/web/mgmt/2019-08-01/web" autorestAzure "github.com/Azure/go-autorest/autorest/azure" ) @@ -771,6 +772,64 @@ func CreateFrontDoorFrontendEndpointClientE(subscriptionID string) (*frontdoor.F return &client, nil } +// CreateSynapseWorkspaceClientE is a helper function that will setup a synapse client. +func CreateSynapseWorkspaceClientE(subscriptionID string) (*synapse.WorkspacesClient, error) { + // Validate Azure subscription ID + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + + // Lookup environment URI + baseURI, err := getBaseURI() + if err != nil { + return nil, err + } + + // Create a synapse client + synapseWorkspaceClient := synapse.NewWorkspacesClientWithBaseURI(baseURI, subscriptionID) + + // Create an authorizer + authorizer, err := NewAuthorizer() + if err != nil { + return nil, err + } + + // Attach authorizer to the client + synapseWorkspaceClient.Authorizer = *authorizer + + return &synapseWorkspaceClient, nil +} + +// CreateSynapseSqlPoolClientE is a helper function that will setup a synapse client. +func CreateSynapseSqlPoolClientE(subscriptionID string) (*synapse.SQLPoolsClient, error) { + // Validate Azure subscription ID + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + + // Lookup environment URI + baseURI, err := getBaseURI() + if err != nil { + return nil, err + } + + // Create a synapse client + synapseSqlPoolClient := synapse.NewSQLPoolsClientWithBaseURI(baseURI, subscriptionID) + + // Create an authorizer + authorizer, err := NewAuthorizer() + if err != nil { + return nil, err + } + + // Attach authorizer to the client + synapseSqlPoolClient.Authorizer = *authorizer + + return &synapseSqlPoolClient, nil +} + // GetKeyVaultURISuffixE returns the proper KeyVault URI suffix for the configured Azure environment. // This function would fail the test if there is an error. func GetKeyVaultURISuffixE() (string, error) { diff --git a/modules/azure/synapse.go b/modules/azure/synapse.go new file mode 100644 index 000000000..079c1cec9 --- /dev/null +++ b/modules/azure/synapse.go @@ -0,0 +1,63 @@ +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/synapse/mgmt/2020-12-01/synapse" + "github.com/gruntwork-io/terratest/modules/testing" + "github.com/stretchr/testify/require" +) + +// GetSynapseWorkspace is a helper function that gets the synapse workspace. +// This function would fail the test if there is an error. +func GetSynapseWorkspace(t testing.TestingT, resGroupName string, workspaceName string, subscriptionID string) *synapse.Workspace { + Workspace, err := GetSynapseWorkspaceE(t, subscriptionID, resGroupName, workspaceName) + require.NoError(t, err) + + return Workspace +} + +// GetSynapseSqlPool is a helper function that gets the synapse workspace. +// This function would fail the test if there is an error. +func GetSynapseSqlPool(t testing.TestingT, resGroupName string, workspaceName string, sqlPoolName string, subscriptionID string) *synapse.SQLPool { + SQLPool, err := GetSynapseSqlPoolE(t, subscriptionID, resGroupName, workspaceName, sqlPoolName) + require.NoError(t, err) + + return SQLPool +} + +// GetSynapseWorkspaceE is a helper function that gets the workspace. +func GetSynapseWorkspaceE(t testing.TestingT, subscriptionID string, resGroupName string, workspaceName string) (*synapse.Workspace, error) { + // Create a synapse client + synapseClient, err := CreateSynapseWorkspaceClientE(subscriptionID) + if err != nil { + return nil, err + } + + // Get the corresponding synapse workspace + synapseWorkspace, err := synapseClient.Get(context.Background(), resGroupName, workspaceName) + if err != nil { + return nil, err + } + + //Return synapse workspace + return &synapseWorkspace, nil +} + +// GetSynapseSqlPoolE is a helper function that gets the synapse sql pool. +func GetSynapseSqlPoolE(t testing.TestingT, subscriptionID string, resGroupName string, workspaceName string, sqlPoolName string) (*synapse.SQLPool, error) { + // Create a synapse client + synapseSqlPoolClient, err := CreateSynapseSqlPoolClientE(subscriptionID) + if err != nil { + return nil, err + } + + // Get the corresponding synapse workspace + synapseSqlPool, err := synapseSqlPoolClient.Get(context.Background(), resGroupName, workspaceName, sqlPoolName) + if err != nil { + return nil, err + } + + //Return synapse workspace + return &synapseSqlPool, nil +} diff --git a/modules/azure/synapse_test.go b/modules/azure/synapse_test.go new file mode 100644 index 000000000..672530675 --- /dev/null +++ b/modules/azure/synapse_test.go @@ -0,0 +1,35 @@ +package azure + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +/* +The below tests are currently stubbed out, with the expectation that they will throw errors. +If/when CRUD methods are introduced for Azure Synapse, these tests can be extended +*/ + +func TestGetSynapseWorkspaceE(t *testing.T) { + t.Parallel() + + resGroupName := "" + subscriptionID := "" + workspaceName := "" + + _, err := GetSynapseWorkspaceE(t, subscriptionID, resGroupName, workspaceName) + require.Error(t, err) +} + +func TestGetSynapseSqlPoolE(t *testing.T) { + t.Parallel() + + resGroupName := "" + subscriptionID := "" + workspaceName := "" + sqlPoolName := "" + + _, err := GetSynapseSqlPoolE(t, subscriptionID, resGroupName, workspaceName, sqlPoolName) + require.Error(t, err) +} diff --git a/test/azure/terraform_azure_synapse_example_test.go b/test/azure/terraform_azure_synapse_example_test.go new file mode 100644 index 000000000..c779115b6 --- /dev/null +++ b/test/azure/terraform_azure_synapse_example_test.go @@ -0,0 +1,65 @@ +//go:build azure +// +build azure + +// NOTE: We use build tags to differentiate azure testing because we currently do not have azure access setup for +// CircleCI. + +package test + +import ( + "strings" + "testing" + + "github.com/gruntwork-io/terratest/modules/azure" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" +) + +func TestTerraformAzureSynapseExample(t *testing.T) { + t.Parallel() + + uniquePostfix := strings.ToLower(random.UniqueId()) + expectedSynapseSqlUser := "sqladminuser" + expectedSynapseProvisioningState := "Succeeded" + expectedLocation := "West US2" + expectedSyPoolSkuName := "DW100c" + + // website::tag::1:: Configure Terraform setting up a path to Terraform code. + terraformOptions := &terraform.Options{ + // The path to where our Terraform code is located + TerraformDir: "../../examples/azure/terraform-azure-synapse-example", + Vars: map[string]interface{}{ + "postfix": uniquePostfix, + "synapse_sql_user": expectedSynapseSqlUser, + "location": expectedLocation, + "synapse_sqlpool_sku_name": expectedSyPoolSkuName, + }, + } + + // website::tag::4:: At the end of the test, run `terraform destroy` to clean up any resources that were created + defer terraform.Destroy(t, terraformOptions) + + // website::tag::2:: Run `terraform init` and `terraform apply`. Fail the test if there are any errors. + terraform.InitAndApply(t, terraformOptions) + // terraform.InitE(t, terraformOptions) + + // website::tag::3:: Run `terraform output` to get the values of output variables + expectedResourceGroupName := terraform.Output(t, terraformOptions, "resource_group_name") + expectedSyDLgen2Name := terraform.Output(t, terraformOptions, "synapse_dlgen2_name") + expectedSyWorkspaceName := terraform.Output(t, terraformOptions, "synapse_workspace_name") + expectedSqlPoolName := terraform.Output(t, terraformOptions, "synapse_sqlpool_name") + + // website::tag::4:: Get synapse details and assert them against the terraform output + actualSynapseWorkspace := azure.GetSynapseWorkspace(t, expectedResourceGroupName, expectedSyWorkspaceName, "") + actualSynapseSqlPool := azure.GetSynapseSqlPool(t, expectedResourceGroupName, expectedSyWorkspaceName, expectedSqlPoolName, "") + + assert.Equal(t, expectedSyWorkspaceName, *actualSynapseWorkspace.Name) + assert.Equal(t, expectedSynapseSqlUser, *actualSynapseWorkspace.WorkspaceProperties.SQLAdministratorLogin) + assert.Equal(t, expectedSynapseProvisioningState, *actualSynapseWorkspace.WorkspaceProperties.ProvisioningState) + assert.Equal(t, expectedLocation, *actualSynapseWorkspace.Location) + assert.Equal(t, expectedSyDLgen2Name, *actualSynapseWorkspace.WorkspaceProperties.DefaultDataLakeStorage.Filesystem) + + assert.Equal(t, expectedSqlPoolName, *actualSynapseSqlPool.Name) + assert.Equal(t, expectedSyPoolSkuName, *actualSynapseSqlPool.Sku.Name) +}