Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Storage: Import Support #1816

Merged
merged 9 commits into from
Aug 30, 2018
Merged

Storage: Import Support #1816

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
7ac49f4
Storage Blob: updating the ID to allow importing
tombuildsstuff Aug 23, 2018
5041277
Storage Blob: Import support
tombuildsstuff Aug 23, 2018
12b06b6
Storage Container: updating the ID
tombuildsstuff Aug 23, 2018
e9e4c15
Storage Container: support for import
tombuildsstuff Aug 23, 2018
2c4c145
Including `blob` in the uri
tombuildsstuff Aug 23, 2018
861f867
Storage Queue: fixing the id / supporting import
tombuildsstuff Aug 24, 2018
0b72201
Storage Table: fixing the ID / import support
tombuildsstuff Aug 24, 2018
058e140
Removing an unused comment
tombuildsstuff Aug 28, 2018
8ae12fc
Setting the fields for the Blob/Container resources
tombuildsstuff Aug 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 149 additions & 86 deletions azurerm/resource_arm_storage_blob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,28 @@ import (
"fmt"
"io"
"log"
"net/url"
"os"
"runtime"
"strings"
"sync"

"github.com/Azure/azure-sdk-for-go/storage"
"github.com/Azure/go-autorest/autorest/azure"
"github.com/hashicorp/terraform/helper/schema"
)

func resourceArmStorageBlob() *schema.Resource {
return &schema.Resource{
Create: resourceArmStorageBlobCreate,
Read: resourceArmStorageBlobRead,
Update: resourceArmStorageBlobUpdate,
Exists: resourceArmStorageBlobExists,
Delete: resourceArmStorageBlobDelete,
Create: resourceArmStorageBlobCreate,
Read: resourceArmStorageBlobRead,
Update: resourceArmStorageBlobUpdate,
Delete: resourceArmStorageBlobDelete,
MigrateState: resourceStorageBlobMigrateState,
SchemaVersion: 1,
Importer: &schema.ResourceImporter{
State: schema.ImportStatePassthrough,
},

Schema: map[string]*schema.Schema{
"name": {
Expand Down Expand Up @@ -127,8 +133,8 @@ func validateArmStorageBlobSize(v interface{}, k string) (ws []string, errors []
func validateArmStorageBlobType(v interface{}, k string) (ws []string, errors []error) {
value := strings.ToLower(v.(string))
validTypes := map[string]struct{}{
"block": struct{}{},
"page": struct{}{},
"block": {},
"page": {},
}

if _, ok := validTypes[value]; !ok {
Expand All @@ -140,6 +146,7 @@ func validateArmStorageBlobType(v interface{}, k string) (ws []string, errors []
func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) error {
armClient := meta.(*ArmClient)
ctx := armClient.StopContext
env := armClient.environment

resourceGroupName := d.Get("resource_group_name").(string)
storageAccountName := d.Get("storage_account_name").(string)
Expand All @@ -154,15 +161,16 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro

name := d.Get("name").(string)
blobType := d.Get("type").(string)
cont := d.Get("storage_container_name").(string)
containerName := d.Get("storage_container_name").(string)
sourceUri := d.Get("source_uri").(string)
contentType := d.Get("content_type").(string)

log.Printf("[INFO] Creating blob %q in storage account %q", name, storageAccountName)
log.Printf("[INFO] Creating blob %q in container %q within storage account %q", name, containerName, storageAccountName)
container := blobClient.GetContainerReference(containerName)
blob := container.GetBlobReference(name)

if sourceUri != "" {
options := &storage.CopyOptions{}
container := blobClient.GetContainerReference(cont)
blob := container.GetBlobReference(name)
err := blob.Copy(sourceUri, options)
if err != nil {
return fmt.Errorf("Error creating storage blob on Azure: %s", err)
Expand All @@ -171,8 +179,6 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro
switch strings.ToLower(blobType) {
case "block":
options := &storage.PutBlobOptions{}
container := blobClient.GetContainerReference(cont)
blob := container.GetBlobReference(name)
err := blob.CreateBlockBlob(options)
if err != nil {
return fmt.Errorf("Error creating storage blob on Azure: %s", err)
Expand All @@ -182,7 +188,8 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro
if source != "" {
parallelism := d.Get("parallelism").(int)
attempts := d.Get("attempts").(int)
if err := resourceArmStorageBlobBlockUploadFromSource(cont, name, source, contentType, blobClient, parallelism, attempts); err != nil {

if err := resourceArmStorageBlobBlockUploadFromSource(containerName, name, source, contentType, blobClient, parallelism, attempts); err != nil {
return fmt.Errorf("Error creating storage blob on Azure: %s", err)
}
}
Expand All @@ -191,15 +198,14 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro
if source != "" {
parallelism := d.Get("parallelism").(int)
attempts := d.Get("attempts").(int)
if err := resourceArmStorageBlobPageUploadFromSource(cont, name, source, contentType, blobClient, parallelism, attempts); err != nil {

if err := resourceArmStorageBlobPageUploadFromSource(containerName, name, source, contentType, blobClient, parallelism, attempts); err != nil {
return fmt.Errorf("Error creating storage blob on Azure: %s", err)
}
} else {
size := int64(d.Get("size").(int))
options := &storage.PutBlobOptions{}

container := blobClient.GetContainerReference(cont)
blob := container.GetBlobReference(name)
blob.Properties.ContentLength = size
blob.Properties.ContentType = contentType
err := blob.PutPageBlob(options)
Expand All @@ -210,7 +216,9 @@ func resourceArmStorageBlobCreate(d *schema.ResourceData, meta interface{}) erro
}
}

d.SetId(name)
// gives us https://example.blob.core.windows.net/container/file.vhd
id := fmt.Sprintf("https://%s.blob.%s/%s/%s", storageAccountName, env.StorageEndpointSuffix, containerName, name)
d.SetId(id)
return resourceArmStorageBlobRead(d, meta)
}

Expand Down Expand Up @@ -539,22 +547,30 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro
armClient := meta.(*ArmClient)
ctx := armClient.StopContext

resourceGroupName := d.Get("resource_group_name").(string)
storageAccountName := d.Get("storage_account_name").(string)
id, err := parseStorageBlobID(d.Id(), armClient.environment)
if err != nil {
return err
}

blobClient, accountExists, err := armClient.getBlobStorageClientForStorageAccount(ctx, resourceGroupName, storageAccountName)
resourceGroup, err := determineResourceGroupForStorageAccount(id.storageAccountName, armClient)
if err != nil {
return err
}

if resourceGroup == nil {
return fmt.Errorf("Unable to determine Resource Group for Storage Account %q", id.storageAccountName)
}

blobClient, accountExists, err := armClient.getBlobStorageClientForStorageAccount(ctx, *resourceGroup, id.storageAccountName)
if err != nil {
return fmt.Errorf("Error getting storage account %s: %+v", storageAccountName, err)
return fmt.Errorf("Error getting storage account %s: %+v", id.storageAccountName, err)
}
if !accountExists {
return fmt.Errorf("Storage account %s not found in resource group %s", storageAccountName, resourceGroupName)
return fmt.Errorf("Storage account %s not found in resource group %s", id.storageAccountName, *resourceGroup)
}

name := d.Get("name").(string)
storageContainerName := d.Get("storage_container_name").(string)

container := blobClient.GetContainerReference(storageContainerName)
blob := container.GetBlobReference(name)
container := blobClient.GetContainerReference(id.containerName)
blob := container.GetBlobReference(id.blobName)

if d.HasChange("content_type") {
blob.Properties.ContentType = d.Get("content_type").(string)
Expand All @@ -563,7 +579,7 @@ func resourceArmStorageBlobUpdate(d *schema.ResourceData, meta interface{}) erro
options := &storage.SetBlobPropertiesOptions{}
err = blob.SetProperties(options)
if err != nil {
return fmt.Errorf("Error setting properties of blob %s (container %s, storage account %s): %+v", name, storageContainerName, storageAccountName, err)
return fmt.Errorf("Error setting properties of blob %s (container %s, storage account %s): %+v", id.blobName, id.containerName, id.storageAccountName, err)
}

return nil
Expand All @@ -573,115 +589,162 @@ func resourceArmStorageBlobRead(d *schema.ResourceData, meta interface{}) error
armClient := meta.(*ArmClient)
ctx := armClient.StopContext

resourceGroupName := d.Get("resource_group_name").(string)
storageAccountName := d.Get("storage_account_name").(string)
id, err := parseStorageBlobID(d.Id(), armClient.environment)
if err != nil {
return err
}

blobClient, accountExists, err := armClient.getBlobStorageClientForStorageAccount(ctx, resourceGroupName, storageAccountName)
resourceGroup, err := determineResourceGroupForStorageAccount(id.storageAccountName, armClient)
if err != nil {
return err
}

if resourceGroup == nil {
return fmt.Errorf("Unable to determine Resource Group for Storage Account %q", id.storageAccountName)
}

blobClient, accountExists, err := armClient.getBlobStorageClientForStorageAccount(ctx, *resourceGroup, id.storageAccountName)
if err != nil {
return err
}
if !accountExists {
log.Printf("[DEBUG] Storage account %q not found, removing blob %q from state", storageAccountName, d.Id())
log.Printf("[DEBUG] Storage account %q not found, removing blob %q from state", id.storageAccountName, d.Id())
d.SetId("")
return nil
}

exists, err := resourceArmStorageBlobExists(d, meta)
log.Printf("[INFO] Checking for existence of storage blob %q in container %q.", id.blobName, id.containerName)
container := blobClient.GetContainerReference(id.containerName)
blob := container.GetBlobReference(id.blobName)
exists, err := blob.Exists()
if err != nil {
return err
return fmt.Errorf("error checking for existence of storage blob %q: %s", id.blobName, err)
}

if !exists {
// Exists already removed this from state
log.Printf("[INFO] Storage blob %q no longer exists, removing from state...", id.blobName)
d.SetId("")
return nil
}

name := d.Get("name").(string)
storageContainerName := d.Get("storage_container_name").(string)

container := blobClient.GetContainerReference(storageContainerName)
blob := container.GetBlobReference(name)

options := &storage.GetBlobPropertiesOptions{}
err = blob.GetProperties(options)
if err != nil {
return fmt.Errorf("Error getting properties of blob %s (container %s, storage account %s): %+v", name, storageContainerName, storageAccountName, err)
return fmt.Errorf("Error getting properties of blob %s (container %s, storage account %s): %+v", id.blobName, id.containerName, id.storageAccountName, err)
}

d.Set("name", id.blobName)
d.Set("storage_container_name", id.containerName)
d.Set("storage_account_name", id.storageAccountName)
d.Set("resource_group_name", resourceGroup)

d.Set("content_type", blob.Properties.ContentType)

d.Set("source_uri", blob.Properties.CopySource)

blobType := strings.ToLower(strings.Replace(string(blob.Properties.BlobType), "Blob", "", 1))
d.Set("type", blobType)

url := blob.GetURL()
if url == "" {
log.Printf("[INFO] URL for %q is empty", name)
log.Printf("[INFO] URL for %q is empty", id.blobName)
}
d.Set("url", url)

return nil
}

func resourceArmStorageBlobExists(d *schema.ResourceData, meta interface{}) (bool, error) {
func resourceArmStorageBlobDelete(d *schema.ResourceData, meta interface{}) error {
armClient := meta.(*ArmClient)
ctx := armClient.StopContext

resourceGroupName := d.Get("resource_group_name").(string)
storageAccountName := d.Get("storage_account_name").(string)
id, err := parseStorageBlobID(d.Id(), armClient.environment)
if err != nil {
return err
}

blobClient, accountExists, err := armClient.getBlobStorageClientForStorageAccount(ctx, resourceGroupName, storageAccountName)
resourceGroup, err := determineResourceGroupForStorageAccount(id.storageAccountName, armClient)
if err != nil {
return false, err
return fmt.Errorf("Unable to determine Resource Group for Storage Account %q: %+v", id.storageAccountName, err)
}
if !accountExists {
log.Printf("[DEBUG] Storage account %q not found, removing blob %q from state", storageAccountName, d.Id())
d.SetId("")
return false, nil
if resourceGroup == nil {
log.Printf("[INFO] Resource Group doesn't exist so the blob won't exist")
return nil
}

name := d.Get("name").(string)
storageContainerName := d.Get("storage_container_name").(string)

log.Printf("[INFO] Checking for existence of storage blob %q.", name)
container := blobClient.GetContainerReference(storageContainerName)
blob := container.GetBlobReference(name)
exists, err := blob.Exists()
blobClient, accountExists, err := armClient.getBlobStorageClientForStorageAccount(ctx, *resourceGroup, id.storageAccountName)
if err != nil {
return false, fmt.Errorf("error testing existence of storage blob %q: %s", name, err)
return err
}
if !accountExists {
log.Printf("[INFO] Storage Account %q doesn't exist so the blob won't exist", id.storageAccountName)
return nil
}

if !exists {
log.Printf("[INFO] Storage blob %q no longer exists, removing from state...", name)
d.SetId("")
log.Printf("[INFO] Deleting storage blob %q", id.blobName)
options := &storage.DeleteBlobOptions{}
container := blobClient.GetContainerReference(id.containerName)
blob := container.GetBlobReference(id.blobName)
_, err = blob.DeleteIfExists(options)
if err != nil {
return fmt.Errorf("Error deleting storage blob %q: %s", id.blobName, err)
}

return exists, nil
return nil
}

func resourceArmStorageBlobDelete(d *schema.ResourceData, meta interface{}) error {
armClient := meta.(*ArmClient)
ctx := armClient.StopContext

resourceGroupName := d.Get("resource_group_name").(string)
storageAccountName := d.Get("storage_account_name").(string)
type storageBlobId struct {
storageAccountName string
containerName string
blobName string
}

blobClient, accountExists, err := armClient.getBlobStorageClientForStorageAccount(ctx, resourceGroupName, storageAccountName)
func parseStorageBlobID(input string, environment azure.Environment) (*storageBlobId, error) {
uri, err := url.Parse(input)
if err != nil {
return err
return nil, fmt.Errorf("Error parsing %q as URI: %+v", input, err)
}
if !accountExists {
log.Printf("[INFO]Storage Account %q doesn't exist so the blob won't exist", storageAccountName)
return nil

// trim the leading `/`
segments := strings.Split(strings.TrimPrefix(uri.Path, "/"), "/")
if len(segments) < 2 {
return nil, fmt.Errorf("Expected number of segments in the path to be < 2 but got %d", len(segments))
}

name := d.Get("name").(string)
storageContainerName := d.Get("storage_container_name").(string)
storageAccountName := strings.Replace(uri.Host, fmt.Sprintf(".blob.%s", environment.StorageEndpointSuffix), "", 1)
containerName := segments[0]
blobName := strings.TrimPrefix(uri.Path, fmt.Sprintf("/%s/", containerName))

log.Printf("[INFO] Deleting storage blob %q", name)
options := &storage.DeleteBlobOptions{}
container := blobClient.GetContainerReference(storageContainerName)
blob := container.GetBlobReference(name)
_, err = blob.DeleteIfExists(options)
id := storageBlobId{
storageAccountName: storageAccountName,
containerName: containerName,
blobName: blobName,
}
return &id, nil
}

func determineResourceGroupForStorageAccount(accountName string, client *ArmClient) (*string, error) {
storageClient := client.storageServiceClient
ctx := client.StopContext

// first locate which resource group the storage account is in
groupsResp, err := storageClient.List(ctx)
if err != nil {
return fmt.Errorf("Error deleting storage blob %q: %s", name, err)
return nil, fmt.Errorf("Error loading the Resource Groups for Storage Account %q: %+v", accountName, err)
}

d.SetId("")
return nil
if groups := groupsResp.Value; groups != nil {
for _, group := range *groups {
if group.Name != nil && *group.Name == accountName {
groupId, err := parseAzureResourceID(*group.ID)
if err != nil {
return nil, err
}

return &groupId.ResourceGroup, nil
}
}
}

return nil, nil
}
Loading