From 1a5b3ae727865c41751810546769760726e66941 Mon Sep 17 00:00:00 2001 From: The Skunk Lab Date: Thu, 11 Feb 2021 09:36:58 -0600 Subject: [PATCH] feat: adding Azure Storage module (#638) * storage module * update custom error * move unit test update prefix * rm azure ref * refactor test * chg docs * chg comment * formatting * add comment * cleanup nits * chg 4 consistency * mv comments * refactor pattern * terraform * fix dns method * fix pin comments. * Fix pin issues. * spelling. * Add doc ref. * address PR comments Co-authored-by: wmattlong Co-authored-by: richard guthrie Co-authored-by: richard guthrie Co-authored-by: Richard Guthrie Co-authored-by: Hadwa Abdelhalem --- .../terraform-azure-storage-example/README.md | 43 +++ .../terraform-azure-storage-example/main.tf | 53 ++++ .../outputs.tf | 19 ++ .../variables.tf | 56 ++++ modules/azure/storage.go | 245 ++++++++++++++++++ modules/azure/storage_test.go | 42 +++ .../terraform_azure_storage_example_test.go | 70 +++++ 7 files changed, 528 insertions(+) create mode 100644 examples/azure/terraform-azure-storage-example/README.md create mode 100644 examples/azure/terraform-azure-storage-example/main.tf create mode 100644 examples/azure/terraform-azure-storage-example/outputs.tf create mode 100644 examples/azure/terraform-azure-storage-example/variables.tf create mode 100644 modules/azure/storage.go create mode 100644 modules/azure/storage_test.go create mode 100644 test/azure/terraform_azure_storage_example_test.go diff --git a/examples/azure/terraform-azure-storage-example/README.md b/examples/azure/terraform-azure-storage-example/README.md new file mode 100644 index 000000000..a04202203 --- /dev/null +++ b/examples/azure/terraform-azure-storage-example/README.md @@ -0,0 +1,43 @@ +# Terraform Azure Storage Example + +This folder contains a simple Terraform module that deploys resources in [Azure](https://azure.microsoft.com/) to demonstrate +how you can use TerraTest to write automated tests for your Azure Terraform code. This module deploys a +Storage Account. + +- An [Azure Storage Account](https://azure.microsoft.com/services/storage/) that gives the module the following: + - [Stock Account Name](https://azure.microsoft.com/services/storage/) with the value specified in the `storage_account_name` output variable. + - [Storage Account Tier](https://azure.microsoft.com/services/storage/) with the value specified in the `"storage_account_account_tier` output variable. + - [Storage Account Kind](https://azure.microsoft.com/services/storage/) with the value specified in the `"storage_account_account_kind` output variable. + - [Storage Container](https://azure.microsoft.com/services/storage/) with the value specified in the `"storage_container_name` output variable. + +Check out [test/azure/terraform_azure_storage_example_test.go](/test/azure/terraform_azure_storage_example_test.go) to see how you can write +automated tests for this module. + +Note that the Storage Account in this module don't actually do anything; it just runs the resources for +demonstration purposes. + +**WARNING**: This module and the automated tests for it deploy real resources into your Azure account which can cost you +money. The resources are all part of the [Azure Free Account](https://azure.microsoft.com/free/), so if you haven't used that up, +it should be free, but you are completely responsible for all Azure charges. + +## Running this module manually + +1. Sign up for [Azure](https://azure.microsoft.com/) +1. Configure your Azure credentials using one of the [supported methods for Azure CLI + tools](https://docs.microsoft.com/cli/azure/azure-cli-configuration?view=azure-cli-latest) +1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH` +1. Ensure [environment variables](../README.md#review-environment-variables) are available +1. Run `terraform init` +1. Run `terraform apply` +1. When you're done, run `terraform destroy` + +## Running automated tests against this module + +1. Sign up for [Azure](https://azure.microsoft.com/) +1. Configure your Azure credentials using one of the [supported methods for Azure CLI + tools](https://docs.microsoft.com/cli/azure/azure-cli-configuration?view=azure-cli-latest) +1. Install [Terraform](https://www.terraform.io/) and make sure it's on your `PATH` +1. Configure your TerraTest [Go test environment](../README.md) +1. `cd test/azure` +1. `go build terraform_azure_storage_example_test.go` +1. `go test -v -run TestTerraformAzureStorageExample` diff --git a/examples/azure/terraform-azure-storage-example/main.tf b/examples/azure/terraform-azure-storage-example/main.tf new file mode 100644 index 000000000..dd4654e38 --- /dev/null +++ b/examples/azure/terraform-azure-storage-example/main.tf @@ -0,0 +1,53 @@ +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A STORAGE ACCOUNT SET +# This is an example of how to deploy a Storage Account. +# --------------------------------------------------------------------------------------------------------------------- +# See test/azure/terraform_azure_storage_example_test.go for how to write automated tests for this code. +# --------------------------------------------------------------------------------------------------------------------- + +provider "azurerm" { + version = "~> 2.20" + features {} +} + +# PIN TERRAFORM VERSION + +terraform { + # This module is now only being tested with Terraform 0.13.x. However, to make upgrading easier, we are setting + # 0.12.26 as the minimum version, as that version added support for required_providers with source URLs, making it + # forwards compatible with 0.13.x code. + required_version = ">= 0.12.26" +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A RESOURCE GROUP +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_resource_group" "resource_group" { + name = "terratest-storage-rg-${var.postfix}" + location = var.location +} + +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY A STORAGE ACCOUNT +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_storage_account" "storage_account" { + name = "storage${var.postfix}" + resource_group_name = azurerm_resource_group.resource_group.name + location = azurerm_resource_group.resource_group.location + account_kind = var.storage_account_kind + account_tier = var.storage_account_tier + account_replication_type = var.storage_replication_type +} + +# --------------------------------------------------------------------------------------------------------------------- +# ADD A CONTAINER TO THE STORAGE ACCOUNT +# --------------------------------------------------------------------------------------------------------------------- + +resource "azurerm_storage_container" "container" { + name = "container1" + storage_account_name = azurerm_storage_account.storage_account.name + container_access_type = var.container_access_type +} + diff --git a/examples/azure/terraform-azure-storage-example/outputs.tf b/examples/azure/terraform-azure-storage-example/outputs.tf new file mode 100644 index 000000000..eb0d8481c --- /dev/null +++ b/examples/azure/terraform-azure-storage-example/outputs.tf @@ -0,0 +1,19 @@ +output "resource_group_name" { + value = azurerm_resource_group.resource_group.name +} + +output "storage_account_name" { + value = azurerm_storage_account.storage_account.name +} + +output "storage_account_account_tier" { + value = azurerm_storage_account.storage_account.account_tier +} + +output "storage_account_account_kind" { + value = azurerm_storage_account.storage_account.account_kind +} + +output "storage_container_name" { + value = azurerm_storage_container.container.name +} diff --git a/examples/azure/terraform-azure-storage-example/variables.tf b/examples/azure/terraform-azure-storage-example/variables.tf new file mode 100644 index 000000000..ac98ff640 --- /dev/null +++ b/examples/azure/terraform-azure-storage-example/variables.tf @@ -0,0 +1,56 @@ +# --------------------------------------------------------------------------------------------------------------------- +# ENVIRONMENT VARIABLES +# Define these secrets as environment variables +# --------------------------------------------------------------------------------------------------------------------- + +# ARM_CLIENT_ID +# ARM_CLIENT_SECRET +# ARM_SUBSCRIPTION_ID +# ARM_TENANT_ID + +# --------------------------------------------------------------------------------------------------------------------- +# REQUIRED PARAMETERS +# You must provide a value for each of these parameters. +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# OPTIONAL PARAMETERS +# These parameters have reasonable defaults. +# --------------------------------------------------------------------------------------------------------------------- + +variable "location" { + description = "The location to set for the storage account." + type = string + default = "East US" +} + +variable "storage_account_kind" { + description = "The kind of storage account to set" + type = string + default = "StorageV2" +} + +variable "storage_account_tier" { + description = "The tier of storage account to set" + type = string + default = "Standard" +} + +variable "storage_replication_type" { + description = "The replication type of storage account to set" + type = string + default = "GRS" +} + +variable "container_access_type" { + description = "The replication type of storage account to set" + type = string + default = "private" +} + +variable "postfix" { + description = "A postfix string to centrally mitigate resource name collisions" + type = string + default = "resource" +} + diff --git a/modules/azure/storage.go b/modules/azure/storage.go new file mode 100644 index 000000000..7db176ee6 --- /dev/null +++ b/modules/azure/storage.go @@ -0,0 +1,245 @@ +package azure + +import ( + "context" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/services/storage/mgmt/2019-06-01/storage" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/stretchr/testify/require" +) + +// StorageAccountExists indicates whether the storage account name exactly matches; otherwise false. +// This function would fail the test if there is an error. +func StorageAccountExists(t *testing.T, storageAccountName string, resourceGroupName string, subscriptionID string) bool { + result, err := StorageAccountExistsE(storageAccountName, resourceGroupName, subscriptionID) + require.NoError(t, err) + return result +} + +// StorageBlobContainerExists returns true if the container name exactly matches; otherwise false +// This function would fail the test if there is an error. +func StorageBlobContainerExists(t *testing.T, containerName string, storageAccountName string, resourceGroupName string, subscriptionID string) bool { + result, err := StorageBlobContainerExistsE(containerName, storageAccountName, resourceGroupName, subscriptionID) + require.NoError(t, err) + return result +} + +// GetStorageBlobContainerPublicAccess indicates whether a storage container has public access; otherwise false. +// This function would fail the test if there is an error. +func GetStorageBlobContainerPublicAccess(t *testing.T, containerName string, storageAccountName string, resourceGroupName string, subscriptionID string) bool { + result, err := GetStorageBlobContainerPublicAccessE(containerName, storageAccountName, resourceGroupName, subscriptionID) + require.NoError(t, err) + return result +} + +// GetStorageAccountKind returns one of Storage, StorageV2, BlobStorage, FileStorage, or BlockBlobStorage. +// This function would fail the test if there is an error. +func GetStorageAccountKind(t *testing.T, storageAccountName string, resourceGroupName string, subscriptionID string) string { + result, err := GetStorageAccountKindE(storageAccountName, resourceGroupName, subscriptionID) + require.NoError(t, err) + return result +} + +// GetStorageAccountSkuTier returns the storage account sku tier as Standard or Premium. +// This function would fail the test if there is an error. +func GetStorageAccountSkuTier(t *testing.T, storageAccountName string, resourceGroupName string, subscriptionID string) string { + result, err := GetStorageAccountSkuTierE(storageAccountName, resourceGroupName, subscriptionID) + require.NoError(t, err) + return result +} + +// GetStorageDNSString builds and returns the storage account dns string if the storage account exists. +// This function would fail the test if there is an error. +func GetStorageDNSString(t *testing.T, storageAccountName string, resourceGroupName string, subscriptionID string) string { + result, err := GetStorageDNSStringE(storageAccountName, resourceGroupName, subscriptionID) + require.NoError(t, err) + return result +} + +// StorageAccountExistsE indicates whether the storage account name exists; otherwise false. +func StorageAccountExistsE(storageAccountName, resourceGroupName, subscriptionID string) (bool, error) { + _, err := GetStorageAccountE(storageAccountName, resourceGroupName, subscriptionID) + if err != nil { + if ResourceNotFoundErrorExists(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// GetStorageAccountE gets a storage account; otherwise error. See https://docs.microsoft.com/rest/api/storagerp/storageaccounts/getproperties for more information. +func GetStorageAccountE(storageAccountName, resourceGroupName, subscriptionID string) (*storage.Account, error) { + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + resourceGroupName, err2 := getTargetAzureResourceGroupName((resourceGroupName)) + if err2 != nil { + return nil, err2 + } + storageAccount, err3 := GetStorageAccountPropertyE(storageAccountName, resourceGroupName, subscriptionID) + if err3 != nil { + return nil, err3 + } + return storageAccount, nil +} + +// StorageBlobContainerExistsE returns true if the container name exists; otherwise false. +func StorageBlobContainerExistsE(containerName, storageAccountName, resourceGroupName, subscriptionID string) (bool, error) { + _, err := GetStorageBlobContainerE(containerName, storageAccountName, resourceGroupName, subscriptionID) + if err != nil { + if ResourceNotFoundErrorExists(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// GetStorageBlobContainerPublicAccessE indicates whether a storage container has public access; otherwise false. +func GetStorageBlobContainerPublicAccessE(containerName, storageAccountName, resourceGroupName, subscriptionID string) (bool, error) { + container, err := GetStorageBlobContainerE(containerName, storageAccountName, resourceGroupName, subscriptionID) + if err != nil { + if ResourceNotFoundErrorExists(err) { + return false, nil + } + return false, err + } + + return (string(container.PublicAccess) != "None"), nil +} + +// GetStorageAccountKindE returns one of Storage, StorageV2, BlobStorage, FileStorage, or BlockBlobStorage. +func GetStorageAccountKindE(storageAccountName, resourceGroupName, subscriptionID string) (string, error) { + + storageAccount, err := GetStorageAccountPropertyE(storageAccountName, resourceGroupName, subscriptionID) + if err != nil { + return "", err + } + return string(storageAccount.Kind), nil +} + +// GetStorageAccountSkuTierE returns the storage account sku tier as Standard or Premium. +func GetStorageAccountSkuTierE(storageAccountName, resourceGroupName, subscriptionID string) (string, error) { + storageAccount, err := GetStorageAccountPropertyE(storageAccountName, resourceGroupName, subscriptionID) + if err != nil { + return "", err + } + return string(storageAccount.Sku.Tier), nil +} + +// GetStorageBlobContainerE returns Blob container client. +func GetStorageBlobContainerE(containerName, storageAccountName, resourceGroupName, subscriptionID string) (*storage.BlobContainer, error) { + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + resourceGroupName, err2 := getTargetAzureResourceGroupName((resourceGroupName)) + if err2 != nil { + return nil, err2 + } + client, err := GetStorageBlobContainerClientE(subscriptionID) + if err != nil { + return nil, err + } + container, err := client.Get(context.Background(), resourceGroupName, storageAccountName, containerName) + if err != nil { + return nil, err + } + return &container, nil +} + +// GetStorageAccountPropertyE returns StorageAccount properties. +func GetStorageAccountPropertyE(storageAccountName, resourceGroupName, subscriptionID string) (*storage.Account, error) { + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + resourceGroupName, err2 := getTargetAzureResourceGroupName((resourceGroupName)) + if err2 != nil { + return nil, err2 + } + client, err := GetStorageAccountClientE(subscriptionID) + if err != nil { + return nil, err + } + account, err := client.GetProperties(context.Background(), resourceGroupName, storageAccountName, "") + if err != nil { + return nil, err + } + return &account, nil +} + +// GetStorageAccountClientE creates a storage account client. +func GetStorageAccountClientE(subscriptionID string) (*storage.AccountsClient, error) { + // Validate Azure subscription ID + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + + storageAccountClient := storage.NewAccountsClient(subscriptionID) + authorizer, err := NewAuthorizer() + if err != nil { + return nil, err + } + storageAccountClient.Authorizer = *authorizer + return &storageAccountClient, nil +} + +// GetStorageBlobContainerClientE creates a storage container client. +func GetStorageBlobContainerClientE(subscriptionID string) (*storage.BlobContainersClient, error) { + subscriptionID, err := getTargetAzureSubscription(subscriptionID) + if err != nil { + return nil, err + } + + blobContainerClient := storage.NewBlobContainersClient(subscriptionID) + authorizer, err := NewAuthorizer() + + if err != nil { + return nil, err + } + blobContainerClient.Authorizer = *authorizer + return &blobContainerClient, nil +} + +// GetStorageURISuffixE returns the proper storage URI suffix for the configured Azure environment. +func GetStorageURISuffixE() (string, error) { + envName := "AzurePublicCloud" + env, err := azure.EnvironmentFromName(envName) + if err != nil { + return "", err + } + return env.StorageEndpointSuffix, nil +} + +// GetStorageAccountPrimaryBlobEndpointE gets the storage account blob endpoint as URI string. +func GetStorageAccountPrimaryBlobEndpointE(storageAccountName, resourceGroupName, subscriptionID string) (string, error) { + storageAccount, err := GetStorageAccountPropertyE(storageAccountName, resourceGroupName, subscriptionID) + if err != nil { + return "", err + } + + return *storageAccount.AccountProperties.PrimaryEndpoints.Blob, nil +} + +// GetStorageDNSStringE builds and returns the storage account dns string if the storage account exists. +func GetStorageDNSStringE(storageAccountName, resourceGroupName, subscriptionID string) (string, error) { + retval, err := StorageAccountExistsE(storageAccountName, resourceGroupName, subscriptionID) + if err != nil { + return "", err + } + if retval { + storageSuffix, err2 := GetStorageURISuffixE() + if err2 != nil { + return "", err2 + } + return fmt.Sprintf("https://%s.blob.%s/", storageAccountName, storageSuffix), nil + } + + return "", NewNotFoundError("storage account", storageAccountName, "") +} diff --git a/modules/azure/storage_test.go b/modules/azure/storage_test.go new file mode 100644 index 000000000..045114a51 --- /dev/null +++ b/modules/azure/storage_test.go @@ -0,0 +1,42 @@ +package azure + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +/* +The below tests are currently stubbed out, with the expectation that they will throw errors. +If/when methods to create and delete storage accounts are added, these tests can be extended. +*/ + +func TestStorageAccountExists(t *testing.T) { + _, err := StorageAccountExistsE("", "", "") + require.Error(t, err) +} + +func TestStorageBlobContainerExists(t *testing.T) { + _, err := StorageBlobContainerExistsE("", "", "", "") + require.Error(t, err) +} + +func TestStorageBlobContainerPublicAccess(t *testing.T) { + _, err := GetStorageBlobContainerPublicAccessE("", "", "", "") + require.Error(t, err) +} + +func TestGetStorageAccountKind(t *testing.T) { + _, err := GetStorageAccountKindE("", "", "") + require.Error(t, err) +} + +func TestGetStorageAccountSkuTier(t *testing.T) { + _, err := GetStorageAccountSkuTierE("", "", "") + require.Error(t, err) +} + +func TestGetStorageDNSString(t *testing.T) { + _, err := GetStorageDNSStringE("", "", "") + require.Error(t, err) +} diff --git a/test/azure/terraform_azure_storage_example_test.go b/test/azure/terraform_azure_storage_example_test.go new file mode 100644 index 000000000..ec71d0fb3 --- /dev/null +++ b/test/azure/terraform_azure_storage_example_test.go @@ -0,0 +1,70 @@ +// +build azure + +// NOTE: We use build tags to differentiate azure testing because we currently do not have azure access setup for +// CircleCI. + +package test + +import ( + "fmt" + "strings" + "testing" + + "github.com/gruntwork-io/terratest/modules/azure" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/stretchr/testify/assert" +) + +func TestTerraformAzureStorageExample(t *testing.T) { + t.Parallel() + + // subscriptionID is overridden by the environment variable "ARM_SUBSCRIPTION_ID" + subscriptionID := "" + uniquePostfix := random.UniqueId() + + // website::tag::1:: Configure Terraform setting up a path to Terraform code. + terraformOptions := &terraform.Options{ + // The path to where our Terraform code is located + TerraformDir: "../../examples/azure/terraform-azure-storage-example", + + // Variables to pass to our Terraform code using -var options + Vars: map[string]interface{}{ + "postfix": strings.ToLower(uniquePostfix), + }, + } + + // website::tag::4:: At the end of the test, run `terraform destroy` to clean up any resources that were created + defer terraform.Destroy(t, terraformOptions) + + // website::tag::2:: Run `terraform init` and `terraform apply`. Fail the test if there are any errors. + terraform.InitAndApply(t, terraformOptions) + + // website::tag::3:: Run `terraform output` to get the values of output variables + resourceGroupName := terraform.Output(t, terraformOptions, "resource_group_name") + storageAccountName := terraform.Output(t, terraformOptions, "storage_account_name") + storageAccountTier := terraform.Output(t, terraformOptions, "storage_account_account_tier") + storageAccountKind := terraform.Output(t, terraformOptions, "storage_account_account_kind") + storageBlobContainerName := terraform.Output(t, terraformOptions, "storage_container_name") + + // website::tag::4:: Verify storage account properties and ensure it matches the output. + storageAccountExists := azure.StorageAccountExists(t, storageAccountName, resourceGroupName, subscriptionID) + assert.True(t, storageAccountExists, "storage account does not exist") + + containerExists := azure.StorageBlobContainerExists(t, storageBlobContainerName, storageAccountName, resourceGroupName, subscriptionID) + assert.True(t, containerExists, "storage container does not exist") + + publicAccess := azure.GetStorageBlobContainerPublicAccess(t, storageBlobContainerName, storageAccountName, resourceGroupName, subscriptionID) + assert.False(t, publicAccess, "storage container has public access") + + accountKind := azure.GetStorageAccountKind(t, storageAccountName, resourceGroupName, subscriptionID) + assert.Equal(t, storageAccountKind, accountKind, "storage account kind mismatch") + + skuTier := azure.GetStorageAccountSkuTier(t, storageAccountName, resourceGroupName, subscriptionID) + assert.Equal(t, storageAccountTier, skuTier, "sku tier mismatch") + + actualDNSString := azure.GetStorageDNSString(t, storageAccountName, resourceGroupName, subscriptionID) + storageSuffix, _ := azure.GetStorageURISuffixE() + expectedDNS := fmt.Sprintf("https://%s.blob.%s/", storageAccountName, storageSuffix) + assert.Equal(t, expectedDNS, actualDNSString, "Storage DNS string mismatch") +}