Merge pull request #3964 from ministryofjustice/remove-unused-data-pl…

…atform-resources Remove unused resources
ministryofjustice · Nov 13, 2023 · f10c727 · f10c727
2 parents 8ecd3b2 + 4d66862
commit f10c727
Show file tree

Hide file tree

Showing 8 changed files with 11 additions and 194 deletions.
diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf
@@ -15,7 +15,6 @@ resource "aws_api_gateway_deployment" "deployment" {
     #       resources will show a difference after the initial implementation.
     #       It will stabilize to only change when resources change afterwards.
     redeployment = sha1(jsonencode([
-      aws_api_gateway_resource.get_glue_metadata,
       aws_api_gateway_resource.docs,
       aws_api_gateway_resource.data_product,
       aws_api_gateway_resource.register_data_product,
@@ -27,7 +26,6 @@ resource "aws_api_gateway_deployment" "deployment" {
       aws_api_gateway_resource.schema_for_data_product_table_name,
       aws_api_gateway_method.preview_data_from_data_product,
       aws_api_gateway_method.docs,
-      aws_api_gateway_method.get_glue_metadata,
       aws_api_gateway_method.register_data_product,
       aws_api_gateway_method.upload_data_for_data_product_table_name,
       aws_api_gateway_method.create_schema_for_data_product_table_name,
@@ -39,7 +37,6 @@ resource "aws_api_gateway_deployment" "deployment" {
       aws_api_gateway_integration.upload_data_for_data_product_table_name_to_lambda,
       aws_api_gateway_integration.proxy_to_lambda,
       aws_api_gateway_integration.docs_lambda_root,
-      aws_api_gateway_integration.get_glue_metadata,
       aws_api_gateway_integration.register_data_product_to_lambda,
       aws_api_gateway_integration.create_schema_for_data_product_table_name_to_lambda,
       aws_api_gateway_integration.get_schema_for_data_product_table_name_to_lambda,
@@ -427,42 +424,6 @@ resource "aws_api_gateway_integration" "docs_lambda_root" {
   }
 }
 
-# get_glue_metadata endpoint
-
-resource "aws_api_gateway_resource" "get_glue_metadata" {
-  parent_id   = aws_api_gateway_rest_api.data_platform.root_resource_id
-  path_part   = "get_glue_metadata"
-  rest_api_id = aws_api_gateway_rest_api.data_platform.id
-}
-
-resource "aws_api_gateway_method" "get_glue_metadata" {
-  authorization = "CUSTOM"
-  authorizer_id = aws_api_gateway_authorizer.authorizer.id
-  http_method   = "GET"
-  resource_id   = aws_api_gateway_resource.get_glue_metadata.id
-  rest_api_id   = aws_api_gateway_rest_api.data_platform.id
-
-  request_parameters = {
-    "method.request.header.Authorization" = true,
-    "method.request.querystring.database" = true,
-    "method.request.querystring.table"    = true,
-  }
-}
-
-resource "aws_api_gateway_integration" "get_glue_metadata" {
-  http_method             = aws_api_gateway_method.get_glue_metadata.http_method
-  resource_id             = aws_api_gateway_resource.get_glue_metadata.id
-  rest_api_id             = aws_api_gateway_rest_api.data_platform.id
-  integration_http_method = "POST"
-  type                    = "AWS_PROXY"
-  uri                     = module.data_product_get_glue_metadata_lambda.lambda_function_invoke_arn
-
-  request_parameters = {
-    "integration.request.querystring.database" = "method.request.querystring.database",
-    "integration.request.querystring.table"    = "method.request.querystring.table"
-  }
-}
-
 # Preview data 
 
 # /data-product/{data-product-name}/table/{table-name}/preview resource

diff --git a/terraform/environments/data-platform/application_variables.auto.tfvars.json b/terraform/environments/data-platform/application_variables.auto.tfvars.json
@@ -11,12 +11,6 @@
     "preproduction": "1.0.0",
     "production": "1.0.0"
   },
-  "get_glue_metadata_versions": {
-    "development": "1.1.1",
-    "test": "1.1.1",
-    "preproduction": "1.1.1",
-    "production": "1.1.1"
-  },
   "presigned_url_versions": {
     "development": "1.4.0",
     "test": "1.4.0",

diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf
@@ -18,9 +18,9 @@ data "aws_iam_policy_document" "log_to_bucket" {
 
 data "aws_iam_policy_document" "read_openmetadata_secrets" {
   statement {
-    sid     = "openmetdataSecretsManager"
-    effect  = "Allow"
-    actions = ["secretsmanager:GetSecretValue"]
+    sid       = "openmetdataSecretsManager"
+    effect    = "Allow"
+    actions   = ["secretsmanager:GetSecretValue"]
     resources = [aws_secretsmanager_secret.openmetadata.id]
   }
 }
@@ -178,23 +178,6 @@ data "aws_iam_policy_document" "iam_policy_document_for_authorizer_lambda" {
   ]
 }
 
-data "aws_iam_policy_document" "iam_policy_document_for_get_glue_metadata_lambda" {
-  source_policy_documents = [
-    data.aws_iam_policy_document.log_to_bucket.json,
-    data.aws_iam_policy_document.create_write_lambda_logs.json,
-  ]
-  statement {
-    sid     = "GlueReadOnly"
-    effect  = "Allow"
-    actions = ["glue:GetTable", "glue:GetTables", "glue:GetDatabase", "glue:GetDatabases"]
-    resources = [
-      "arn:aws:glue:${local.region}:${local.account_id}:catalog",
-      "arn:aws:glue:${local.region}:${local.account_id}:database/*",
-      "arn:aws:glue:${local.region}:${local.account_id}:table/*"
-    ]
-  }
-}
-
 data "aws_iam_policy_document" "iam_policy_document_for_presigned_url_lambda" {
   source_policy_documents = [
     data.aws_iam_policy_document.log_to_bucket.json,
@@ -262,30 +245,6 @@ resource "aws_iam_role_policy_attachment" "attach_allow_invoke_authoriser_lambda
 
 # S3 policy
 
-# TO BE REMOVED
-data "aws_iam_policy_document" "data_platform_product_bucket_policy_document" {
-  statement {
-    sid       = "DenyNonFullControlObjects"
-    effect    = "Deny"
-    actions   = ["s3:PutObject"]
-    resources = ["${module.s3-bucket.bucket.arn}/*"]
-
-    principals {
-      identifiers = ["*"]
-      type        = "AWS"
-    }
-
-    condition {
-      test     = "StringNotEquals"
-      variable = "s3:x-amz-acl"
-
-      values = [
-        "bucket-owner-full-control"
-      ]
-    }
-  }
-}
-
 data "aws_iam_policy_document" "data_s3_bucket_policy_document" {
   statement {
     sid       = "DenyNonFullControlObjects"
@@ -623,11 +582,11 @@ data "aws_iam_policy_document" "iam_policy_document_for_preview_data" {
     sid    = "GluePermissions"
     effect = "Allow"
     actions = [
-     	  "glue:GetTable",
-				"glue:GetPartitions",
-				"glue:GetPartition",
-				"glue:GetDatabases",
-				"glue:GetDatabase"
+      "glue:GetTable",
+      "glue:GetPartitions",
+      "glue:GetPartition",
+      "glue:GetDatabases",
+      "glue:GetDatabase"
     ]
     resources = [
       "*"

diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf
@@ -59,34 +59,6 @@ module "data_product_authorizer_lambda" {
 
 }
 
-module "data_product_get_glue_metadata_lambda" {
-  source                         = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1
-  application_name               = "data_product_get_glue_metadata"
-  tags                           = local.tags
-  description                    = "Lambda to retrieve Glue metadata for a specified table in a database"
-  role_name                      = "get_glue_metadata_lambda_role_${local.environment}"
-  policy_json_attached           = true
-  policy_json                    = data.aws_iam_policy_document.iam_policy_document_for_get_glue_metadata_lambda.json
-  function_name                  = "data_product_get_glue_metadata_${local.environment}"
-  create_role                    = true
-  reserved_concurrent_executions = 1
-
-  image_uri    = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-get-glue-metadata-lambda-ecr-repo:${local.get_glue_metadata_version}"
-  timeout      = 600
-  tracing_mode = "Active"
-  memory_size  = 512
-
-  allowed_triggers = {
-
-    AllowExecutionFromAPIGateway = {
-      action     = "lambda:InvokeFunction"
-      principal  = "apigateway.amazonaws.com"
-      source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.get_glue_metadata.http_method}${aws_api_gateway_resource.get_glue_metadata.path}"
-    }
-  }
-
-}
-
 module "data_product_landing_to_raw_lambda" {
   source                         = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1
   application_name               = "data_product_landing_to_raw"

diff --git a/terraform/environments/data-platform/locals.tf b/terraform/environments/data-platform/locals.tf
@@ -28,7 +28,6 @@ locals {
 
   docs_version                          = lookup(var.docs_versions, local.environment)
   authorizer_version                    = lookup(var.authorizer_versions, local.environment)
-  get_glue_metadata_version             = lookup(var.get_glue_metadata_versions, local.environment)
   presigned_url_version                 = lookup(var.presigned_url_versions, local.environment)
   athena_load_version                   = lookup(var.athena_load_versions, local.environment)
   create_metadata_version               = lookup(var.create_metadata_versions, local.environment)
@@ -55,8 +54,8 @@ locals {
     LANDING_ZONE_BUCKET = module.data_landing_s3_bucket.bucket.id
   }
 
-    openmetadata_environment_vars = {
+  openmetadata_environment_vars = {
     OPENMETADATA_JWT_SECRET_ARN = aws_secretsmanager_secret.openmetadata.id
-    OPENMETADATA_DEV_API_URL = "https://catalogue.apps-tools.development.data-platform.service.justice.gov.uk/api"
+    OPENMETADATA_DEV_API_URL    = "https://catalogue.apps-tools.development.data-platform.service.justice.gov.uk/api"
   }
 }
diff --git a/terraform/environments/data-platform/output.tf b/terraform/environments/data-platform/output.tf
@@ -3,10 +3,6 @@ output "docs_endpoint" {
   value = join("", [aws_api_gateway_deployment.deployment.invoke_url, aws_api_gateway_stage.default_stage.stage_name, "/docs/"])
 }
 
-output "get_glue_metadata_endpoint" {
-  value = join("", [aws_api_gateway_deployment.deployment.invoke_url, aws_api_gateway_stage.default_stage.stage_name, "/get_glue_metadata/"])
-}
-
 output "presigned_url_endpoint" {
   value = join("", [aws_api_gateway_deployment.deployment.invoke_url, aws_api_gateway_stage.default_stage.stage_name, "/presigned_url/"])
-}
+}
diff --git a/terraform/environments/data-platform/s3.tf b/terraform/environments/data-platform/s3.tf
@@ -1,63 +1,3 @@
-# TO BE REMOVED
-module "s3-bucket" { #tfsec:ignore:aws-s3-enable-versioning
-  source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0"
-
-  bucket_prefix      = "data-platform-products-${local.environment}"
-  versioning_enabled = false
-  # Refer to the below section "Replication" before enabling replication
-  replication_enabled = false
-  bucket_policy       = [data.aws_iam_policy_document.data_platform_product_bucket_policy_document.json]
-  providers = {
-    # Here we use the default provider Region for replication. Destination buckets can be within the same Region as the
-    # source bucket. On the other hand, if you need to enable cross-region replication, please contact the Modernisation
-    # Platform team to add a new provider for the additional Region.
-    aws.bucket-replication = aws
-  }
-
-  lifecycle_rule = [
-    {
-      id      = "main"
-      enabled = "Enabled"
-      prefix  = ""
-
-      tags = {
-        rule      = "log"
-        autoclean = "true"
-      }
-
-      transition = [
-        {
-          days          = 90
-          storage_class = "STANDARD_IA"
-          }, {
-          days          = 365
-          storage_class = "GLACIER"
-        }
-      ]
-
-      expiration = {
-        days = 730
-      }
-
-      noncurrent_version_transition = [
-        {
-          days          = 90
-          storage_class = "STANDARD_IA"
-          }, {
-          days          = 365
-          storage_class = "GLACIER"
-        }
-      ]
-
-      noncurrent_version_expiration = {
-        days = 730
-      }
-    }
-  ]
-
-  tags = local.tags
-}
-
 module "data_s3_bucket" { #tfsec:ignore:aws-s3-enable-versioning
   source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=v7.0.0"
 

diff --git a/terraform/environments/data-platform/variables.tf b/terraform/environments/data-platform/variables.tf
@@ -6,10 +6,6 @@ variable "authorizer_versions" {
   type = map(any)
 }
 
-variable "get_glue_metadata_versions" {
-  type = map(any)
-}
-
 variable "presigned_url_versions" {
   type = map(any)
 }