Merge pull request #1 from drandell/assets

add assets module and example

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2] - 2022-08-17
+### Added
+- Dataplex Assets Module
+
+### Changed
+- Bumped minimum version of the provider to 4.32.0
+
 ## [0.1] - 2022-08-05
 ### Added
 - Initial Release

README.md

@@ -21,7 +21,7 @@ These sections describe requirements for using this module.
 The following dependencies must be available:
 
 - [Terraform](https://www.terraform.io/downloads.html) >= 1.3.0, < 2.0.0
-- [Terraform Provider for GCP](https://registry.terraform.io/providers/hashicorp/google/latest) >= v4.31.0, < 5.0.0
+- [Terraform Provider for GCP](https://registry.terraform.io/providers/hashicorp/google/latest) >= v4.32.0, < 5.0.0
 
 ### Service Account
 

examples/basic_dataplex_usage/README.md

@@ -1,15 +1,17 @@
-# Basic Data Catalog Usage
+# Basic Dataplex Usage
 
-This example illustrates showcase  the basic dataplex features.
+This example illustrates showcase the basic dataplex features.
 
 It will:
 - Create a lake
 - Create a zone within that lake
+- Create an asset within the zone that is within our lake
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|-------------------|---------|:--------:|
+| assets | List of asset objects to create | <pre>list(object({<br>    id           = string,<br>    lake_id      = string,<br>    zone_id      = string,<br>    display_name = optional(string),<br>    description  = optional(string),<br>    labels       = optional(map(string), {}),<br>    discovery_spec = object({<br>      enabled          = bool,<br>      schedule         = optional(string),<br>      include_patterns = optional(list(string), []),<br>      exclude_patterns = optional(list(string), []),<br>      csv_options = optional(object({<br>        delimiter              = optional(string, ","),<br>        header_rows            = optional(number, 0)<br>        disable_type_inference = optional(bool, false),<br>        encoding               = optional(string, "UTF-8")<br>      }))<br>      json_options = optional(object({<br>        disable_type_inference = optional(bool, false),<br>        encoding               = optional(string, "UTF-8")<br>      })),<br>    }),<br>    resource_spec = object({<br>      name = string,<br>      type = string<br>    })<br>  }))</pre> | `[]` | no |
 | project\_id | The project to host your resources in | `string` | n/a | yes |
 | location | The location to host your resources in | `string` | n/a | yes |
 | lakes | List of lake objects to create | <pre>list(object({<br>    id                 = string,<br>    display_name       = optional(string),<br>    description        = optional(string),<br>    labels             = optional(map(string), {}),<br>    dataproc_metastore = optional(string)<br>  }))</pre> | `[]` | no |
@@ -19,6 +21,8 @@ It will:
 
 | Name | Description |
 |------|-------------|
+| asset\_self\_links | Map of self links to the defined Dataplex assets |
+| asset\_uids | Map of UIDs to the defined Dataplex assets |
 | lake\_self\_links | Map of self links to the defined Dataplex lakes |
 | lake\_uids | Map of UIDs to the defined Dataplex lakes |
 | zone\_self\_links | Map of self links to the defined Dataplex zones |
@@ -30,4 +34,6 @@ See [`terraform.tfvars`](./terraform.tfvars) for some of the example usage.
 
 ## Notes
 
-N/A
+### Assets
+
+Remember the assets you discover must be in the same region(s) as the location policy of the lake/zone you create. 

examples/basic_dataplex_usage/main.tf

@@ -5,4 +5,5 @@ module "google_dataplex" {
   location   = var.location
   lakes      = var.lakes
   zones      = var.zones
+  assets     = var.assets
 }

examples/basic_dataplex_usage/outputs.tf

@@ -1,19 +1,29 @@
+output "asset_self_links" {
+  value       = module.google_dataplex.asset_self_links
+  description = "Map of self links to the defined Dataplex assets"
+}
+
+output "asset_uids" {
+  value       = module.google_dataplex.asset_uids
+  description = "Map of UIDs to the defined Dataplex assets"
+}
+
 output "lake_self_links" {
-  value = module.google_dataplex.lake_self_links
+  value       = module.google_dataplex.lake_self_links
   description = "Map of self links to the defined Dataplex lakes"
 }
 
 output "lake_uids" {
-  value = module.google_dataplex.lake_uids
+  value       = module.google_dataplex.lake_uids
   description = "Map of UIDs to the defined Dataplex lakes"
 }
 
 output "zone_self_links" {
-  value = module.google_dataplex.zone_self_links
+  value       = module.google_dataplex.zone_self_links
   description = "Map of self links to the defined Dataplex zones"
 }
 
 output "zone_uids" {
-  value = module.google_dataplex.zone_uids
+  value       = module.google_dataplex.zone_uids
   description = "Map of UIDs to the defined Dataplex zones"
 }

examples/basic_dataplex_usage/terraform.tfvars

@@ -1,12 +1,14 @@
 project_id = "billing-budgets"
 location   = "us-central1"
+
 lakes = [
   {
     id           = "my-test-lake"
     display_name = "My Amazing Lake"
     description  = "its a big lake"
   }
 ]
+
 zones = [
   {
     id      = "my-test-zone"
@@ -19,4 +21,19 @@ zones = [
       location_type = "SINGLE_REGION"
     }
   }
+]
+
+assets = [
+  {
+    id      = "my-dummy-asset"
+    lake_id = "my-test-lake"
+    zone_id = "my-test-zone"
+    discovery_spec = {
+      enabled = false
+    }
+    resource_spec = {
+      name = "projects/billing-budgets/buckets/this-is-my-bucket-name"
+      type = "STORAGE_BUCKET"
+    }
+  }
 ]

examples/basic_dataplex_usage/variables.tf

@@ -60,4 +60,42 @@ variable "zones" {
     condition     = alltrue([for zone in var.zones : contains(["SINGLE_REGION", "MULTI_REGION", "LOCATION_TYPE_UNSPECIFIED"], zone.resource_spec["location_type"])])
     error_message = "Supported location types are 'SINGLE_REGION', 'MULTI_REGION', 'LOCATION_TYPE_UNSPECIFIED'."
   }
-}
+}
+
+variable "assets" {
+  type = list(object({
+    id           = string,
+    lake_id      = string,
+    zone_id      = string,
+    display_name = optional(string),
+    description  = optional(string),
+    labels       = optional(map(string), {}),
+    discovery_spec = object({
+      enabled          = bool,
+      schedule         = optional(string),
+      include_patterns = optional(list(string), []),
+      exclude_patterns = optional(list(string), []),
+      csv_options = optional(object({
+        delimiter              = optional(string, ","),
+        header_rows            = optional(number, 0)
+        disable_type_inference = optional(bool, false),
+        encoding               = optional(string, "UTF-8")
+      }))
+      json_options = optional(object({
+        disable_type_inference = optional(bool, false),
+        encoding               = optional(string, "UTF-8")
+      })),
+    }),
+    resource_spec = object({
+      name = string,
+      type = string
+    })
+  }))
+  default     = []
+  description = "(Optional) A list of asset objects"
+
+  validation {
+    condition     = alltrue([for asset in var.assets : contains(["STORAGE_BUCKET", "BIGQUERY_DATASET"], asset.resource_spec["type"])])
+    error_message = "Supported resource spec types are 'STORAGE_BUCKET' or 'BIGQUERY_DATASET'"
+  }
+}

main.tf

@@ -18,3 +18,18 @@ module "dataplex_zones" {
     module.dataplex_lakes
   ]
 }
+
+module "dataplex_assets" {
+  source = "./modules/assets"
+
+  project         = var.project_id
+  location        = var.location
+  assets          = var.assets
+  lake_self_links = module.dataplex_lakes.self_links
+  zone_self_links = module.dataplex_zones.self_links
+
+  depends_on = [
+    module.dataplex_lakes,
+    module.dataplex_zones
+  ]
+}

modules/assets/main.tf

@@ -0,0 +1,47 @@
+locals {
+  assets = { for asset in var.assets : asset["id"] => asset }
+}
+
+resource "google_dataplex_asset" "this" {
+  for_each = local.assets
+
+  location      = var.location
+  project       = var.project
+  lake          = var.lake_self_links[each.value["lake_id"]]
+  dataplex_zone = var.zone_self_links[each.value["zone_id"]]
+
+  name         = each.key
+  description  = each.value["description"]
+  display_name = each.value["display_name"]
+  labels       = each.value["labels"]
+
+  discovery_spec {
+    enabled          = each.value["discovery_spec"]["enabled"]
+    include_patterns = each.value["discovery_spec"]["include_patterns"]
+    exclude_patterns = each.value["discovery_spec"]["exclude_patterns"]
+    schedule         = each.value["discovery_spec"]["schedule"]
+
+    dynamic "csv_options" {
+      for_each = each.value["discovery_spec"]["csv_options"] != null ? { "csv_opt" : each.value["discovery_spec"]["csv_options"] } : {}
+      content {
+        delimiter              = csv_options.value["delimiter"]
+        header_rows            = csv_options.value["header_rows"]
+        disable_type_inference = csv_options.value["disable_type_inference"]
+        encoding               = csv_options.value["encoding"]
+      }
+    }
+
+    dynamic "json_options" {
+      for_each = each.value["discovery_spec"]["json_options"] != null ? { "json_opt" : each.value["discovery_spec"]["json_options"] } : {}
+      content {
+        disable_type_inference = json_options.value["disable_type_inference"]
+        encoding               = json_options.value["encoding"]
+      }
+    }
+  }
+
+  resource_spec {
+    name = each.value["resource_spec"]["name"]
+    type = each.value["resource_spec"]["type"]
+  }
+}

modules/assets/outputs.tf

@@ -0,0 +1,9 @@
+output "self_links" {
+  value       = { for key, asset in google_dataplex_asset.this : key => asset.id }
+  description = "Map of self links of our defined assets"
+}
+
+output "uids" {
+  value       = { for key, asset in google_dataplex_asset.this : key => asset.uid }
+  description = "Map of UIDs of our defined assets"
+}

modules/assets/variables.tf

@@ -0,0 +1,59 @@
+variable "project" {
+  type        = string
+  description = "(Required) The project ID to host the resource in"
+}
+
+variable "location" {
+  type        = string
+  description = "(Required) The location to host the resource in"
+}
+
+variable "assets" {
+  type = list(object({
+    id           = string,
+    lake_id      = string,
+    zone_id      = string,
+    display_name = optional(string),
+    description  = optional(string),
+    labels       = optional(map(string), {}),
+    discovery_spec = object({
+      enabled          = bool,
+      schedule         = optional(string),
+      include_patterns = optional(list(string), []),
+      exclude_patterns = optional(list(string), []),
+      csv_options = optional(object({
+        delimiter              = optional(string, ","),
+        header_rows            = optional(number, 0)
+        disable_type_inference = optional(bool, false),
+        encoding               = optional(string, "UTF-8")
+      }))
+      json_options = optional(object({
+        disable_type_inference = optional(bool, false),
+        encoding               = optional(string, "UTF-8")
+      })),
+    }),
+    resource_spec = object({
+      name = string,
+      type = string
+    })
+  }))
+  default     = []
+  description = "(Optional) A list of asset objects"
+
+  validation {
+    condition     = alltrue([for asset in var.assets : contains(["STORAGE_BUCKET", "BIGQUERY_DATASET"], asset.resource_spec["type"])])
+    error_message = "Supported resource spec types are 'STORAGE_BUCKET' or 'BIGQUERY_DATASET'"
+  }
+}
+
+variable "lake_self_links" {
+  type        = map(any)
+  default     = {}
+  description = "(Optional) A map of lake self_links"
+}
+
+variable "zone_self_links" {
+  type        = map(any)
+  default     = {}
+  description = "(Optional) A map of zone self_links"
+}

modules/assets/versions.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 1.3.0, < 2.0.0"
+
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 4.32.0, < 5.0.0"
+    }
+  }
+}

modules/lakes/versions.tf

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.31.0, < 5.0.0"
+      version = ">= 4.32.0, < 5.0.0"
     }
   }
 }

modules/zones/versions.tf

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.31.0, < 5.0.0"
+      version = ">= 4.32.0, < 5.0.0"
     }
   }
 }

outputs.tf

@@ -1,3 +1,13 @@
+output "asset_self_links" {
+  value       = module.dataplex_assets.self_links
+  description = "Map of self links to the defined Dataplex assets"
+}
+
+output "asset_uids" {
+  value       = module.dataplex_assets.uids
+  description = "Map of UIDs to the defined Dataplex assets"
+}
+
 output "lake_self_links" {
   value       = module.dataplex_lakes.self_links
   description = "Map of self links to the defined Dataplex lakes"