From bf2fd20da87880e80cb2029f12e91e2d352428c8 Mon Sep 17 00:00:00 2001 From: Mat Moore Date: Tue, 10 Oct 2023 14:02:48 +0100 Subject: [PATCH 01/13] Add get-schema endpoint --- terraform/environments/data-platform/api.tf | 48 +++++++++++++++---- .../application_variables.auto.tfvars.json | 6 +++ terraform/environments/data-platform/iam.tf | 20 ++++++++ .../environments/data-platform/lambda.tf | 26 ++++++++++ 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf index 022d6ca1444..fa6bc0d4bfb 100644 --- a/terraform/environments/data-platform/api.tf +++ b/terraform/environments/data-platform/api.tf @@ -123,6 +123,13 @@ resource "aws_api_gateway_resource" "upload_data_for_data_product_table_name" { rest_api_id = aws_api_gateway_rest_api.data_platform.id } +# /data-product/{data-product-name}/table/{table-name}/schema resource +resource "aws_api_gateway_resource" "schema_for_data_product_table_name" { + parent_id = aws_api_gateway_resource.data_product_table_name.id + path_part = "schema" + rest_api_id = aws_api_gateway_rest_api.data_platform.id +} + # /data-product/{data-product-name}/table/{table-name}/upload POST method resource "aws_api_gateway_method" "upload_data_for_data_product_table_name" { authorization = "CUSTOM" @@ -153,19 +160,12 @@ resource "aws_api_gateway_integration" "upload_data_for_data_product_table_name_ } } -# /data-product/{data-product-name}/table/{table-name}/schema resource -resource "aws_api_gateway_resource" "create_schema_for_data_product_table_name" { - parent_id = aws_api_gateway_resource.data_product_table_name.id - path_part = "schema" - rest_api_id = aws_api_gateway_rest_api.data_platform.id -} - # /data-product/{data-product-name}/table/{table-name}/schema POST method resource "aws_api_gateway_method" "create_schema_for_data_product_table_name" { authorization = "CUSTOM" authorizer_id = aws_api_gateway_authorizer.authorizer.id http_method = "POST" - resource_id = aws_api_gateway_resource.create_schema_for_data_product_table_name.id + resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id rest_api_id = aws_api_gateway_rest_api.data_platform.id request_parameters = { @@ -178,7 +178,7 @@ resource "aws_api_gateway_method" "create_schema_for_data_product_table_name" { # /data-product/{data-product-name}/table/{table-name}/schema lambda integration resource "aws_api_gateway_integration" "create_schema_for_data_product_table_name_to_lambda" { http_method = aws_api_gateway_method.create_schema_for_data_product_table_name.http_method - resource_id = aws_api_gateway_resource.create_schema_for_data_product_table_name.id + resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id rest_api_id = aws_api_gateway_rest_api.data_platform.id integration_http_method = "POST" type = "AWS_PROXY" @@ -190,6 +190,36 @@ resource "aws_api_gateway_integration" "create_schema_for_data_product_table_nam } } +# /data-product/{data-product-name}/table/{table-name}/schema GET method +resource "aws_api_gateway_method" "get_schema_for_data_product_table_name" { + authorization = "CUSTOM" + authorizer_id = aws_api_gateway_authorizer.authorizer.id + http_method = "GET" + resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id + rest_api_id = aws_api_gateway_rest_api.data_platform.id + + request_parameters = { + "method.request.header.Authorization" = true, + "method.request.path.data-product-name" = true, + "method.request.path.table-name" = true, + } +} + +# /data-product/{data-product-name}/table/{table-name}/schema lambda integration +resource "aws_api_gateway_integration" "get_schema_for_data_product_table_name_to_lambda" { + http_method = aws_api_gateway_method.get_schema_for_data_product_table_name.http_method + resource_id = aws_api_gateway_resource.schema_for_data_product_table_name.id + rest_api_id = aws_api_gateway_rest_api.data_platform.id + integration_http_method = "POST" + type = "AWS_PROXY" + uri = module.get_schema_lambda.lambda_function_invoke_arn + + request_parameters = { + "integration.request.path.data-product-name" = "method.request.path.data-product-name", + "integration.request.path.table-name" = "method.request.path.table-name", + } +} + # API docs endpoint resource "aws_api_gateway_resource" "docs" { diff --git a/terraform/environments/data-platform/application_variables.auto.tfvars.json b/terraform/environments/data-platform/application_variables.auto.tfvars.json index 51911bbcc8d..566b4444e61 100644 --- a/terraform/environments/data-platform/application_variables.auto.tfvars.json +++ b/terraform/environments/data-platform/application_variables.auto.tfvars.json @@ -58,5 +58,11 @@ "test": "1.0.0", "preproduction": "1.0.0", "production": "1.0.0" + }, + "get_schema_versions": { + "development": "1.0.0", + "test": "1.0.0", + "preproduction": "1.0.0", + "production": "1.0.0" } } diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index be8f7b10fad..7d4512b7eff 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -550,6 +550,26 @@ data "aws_iam_policy_document" "iam_policy_document_for_reload_data_product_lamb } } +data "aws_iam_policy_document" "iam_policy_document_for_get_schema_lambda" { + statement { + sid = "s3LogAccess" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:PutObject", + ] + resources = [ + "${module.s3-bucket.bucket.arn}/logs/*" + ] + } + statement { + sid = "LambdaLogGroup" + effect = "Allow" + actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] + resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] + } +} + data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files_lambda" { source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index 0578dde1980..5b97ad25fae 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -294,3 +294,29 @@ module "data_product_create_schema_lambda" { } } + +module "get_schema_lambda" { + source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 + application_name = "get_schema" + tags = local.tags + description = "Fetch the schema for a table from S3" + role_name = "get_schema_role_${local.environment}" + policy_json = data.aws_iam_policy_document.iam_policy_document_for_get_schema_lambda.json + function_name = "get_schema_${local.environment}" + create_role = true + reserved_concurrent_executions = 1 + + image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/get-schema-lambda-ecr-repo:${local.get_schema_version}" + timeout = 600 + tracing_mode = "Active" + memory_size = 512 + + environment_variables = { + LOG_BUCKET = module.s3-bucket.bucket.id + METADATA_BUCKET = module.s3-bucket.bucket.id + RAW_DATA_BUCKET = module.s3-bucket.bucket.id + CURATED_DATA_BUCKET = module.s3-bucket.bucket.id + LANDING_ZONE_BUCKET = module.s3-bucket.bucket.id + } + +} From 7f9e455d4143b03186eac10c3eba3ef415dd20cd Mon Sep 17 00:00:00 2001 From: Mat Moore Date: Tue, 10 Oct 2023 14:10:56 +0100 Subject: [PATCH 02/13] Add temporary read access to s3 bucket This will need to be redone when we separate the data into multiple buckets. --- terraform/environments/data-platform/iam.tf | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 7d4512b7eff..9f4249e5f06 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -598,6 +598,20 @@ data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + + statement { + sid = "s3MetadataAccessPlaceholder" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:ListBucket" + ] + resources = [ + "${module.s3-bucket.bucket.arn}/*", + "${module.s3-bucket.bucket.arn}" + ] + } + statement { sid = "s3MetadataWrite" effect = "Allow" From 61914db1733e31ca72b16f597bd74833fea8f6f8 Mon Sep 17 00:00:00 2001 From: Mat Moore Date: Tue, 10 Oct 2023 14:15:20 +0100 Subject: [PATCH 03/13] Add missing vars --- terraform/environments/data-platform/locals.tf | 1 + terraform/environments/data-platform/variables.tf | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/terraform/environments/data-platform/locals.tf b/terraform/environments/data-platform/locals.tf index 8c811169c51..3d8d917b583 100644 --- a/terraform/environments/data-platform/locals.tf +++ b/terraform/environments/data-platform/locals.tf @@ -34,6 +34,7 @@ locals { create_metadata_version = lookup(var.create_metadata_versions, local.environment) resync_unprocessed_files_version = lookup(var.resync_unprocessed_files_versions, local.environment) reload_data_product_version = lookup(var.reload_data_product_versions, local.environment) + get_schema_version = lookup(var.get_schema_versions, local.environment) create_schema_version = lookup(var.create_schema_versions, local.environment) landing_to_raw_version = lookup(var.landing_to_raw_versions, local.environment) diff --git a/terraform/environments/data-platform/variables.tf b/terraform/environments/data-platform/variables.tf index 4666e7b95f5..6bfa471153e 100644 --- a/terraform/environments/data-platform/variables.tf +++ b/terraform/environments/data-platform/variables.tf @@ -37,3 +37,7 @@ variable "landing_to_raw_versions" { variable "create_schema_versions" { type = map(any) } + +variable "get_schema_versions" { + type = map(any) +} From bb61fc7ee7459b174af397a160965309d2e7b088 Mon Sep 17 00:00:00 2001 From: Mat Moore Date: Tue, 10 Oct 2023 15:53:00 +0100 Subject: [PATCH 04/13] Fix url --- terraform/environments/data-platform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index 5b97ad25fae..bda4a7524af 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -306,7 +306,7 @@ module "get_schema_lambda" { create_role = true reserved_concurrent_executions = 1 - image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/get-schema-lambda-ecr-repo:${local.get_schema_version}" + image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-get-schema-lambda-ecr-repo:${local.get_schema_version}" timeout = 600 tracing_mode = "Active" memory_size = 512 From 1563dfedae8e195398808431052962fdd4498958 Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Wed, 11 Oct 2023 15:54:25 +0100 Subject: [PATCH 05/13] add elements to deploy fix get_schema properties --- terraform/environments/data-platform/api.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf index fa6bc0d4bfb..b93abd60783 100644 --- a/terraform/environments/data-platform/api.tf +++ b/terraform/environments/data-platform/api.tf @@ -23,12 +23,13 @@ resource "aws_api_gateway_deployment" "deployment" { aws_api_gateway_resource.data_product_table, aws_api_gateway_resource.data_product_table_name, aws_api_gateway_resource.upload_data_for_data_product_table_name, - aws_api_gateway_resource.create_schema_for_data_product_table_name, + aws_api_gateway_resource.schema_for_data_product_table_name, aws_api_gateway_method.docs, aws_api_gateway_method.get_glue_metadata, aws_api_gateway_method.register_data_product, aws_api_gateway_method.upload_data_for_data_product_table_name, aws_api_gateway_method.create_schema_for_data_product_table_name, + aws_api_gateway_method.get_schema_for_data_product_table_name, aws_api_gateway_integration.docs_to_lambda, aws_api_gateway_integration.upload_data_for_data_product_table_name_to_lambda, aws_api_gateway_integration.proxy_to_lambda, @@ -36,6 +37,7 @@ resource "aws_api_gateway_deployment" "deployment" { aws_api_gateway_integration.get_glue_metadata, aws_api_gateway_integration.register_data_product_to_lambda, aws_api_gateway_integration.create_schema_for_data_product_table_name_to_lambda, + aws_api_gateway_integration.get_schema_for_data_product_table_name_to_lambda, ])) } From a9c10107c9f7c34d0d350bd9a5155a44f6f874af Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Mon, 16 Oct 2023 11:32:56 +0100 Subject: [PATCH 06/13] fix lambda resource ref --- terraform/environments/data-platform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index bda4a7524af..e6d6acd757e 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -289,7 +289,7 @@ module "data_product_create_schema_lambda" { action = "lambda:InvokeFunction" function_name = "data_product_create_metadata_${local.environment}" principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.create_schema_for_data_product_table_name.path}" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" } } From a8b6d7b50f366b5a5bccc9da4fe7e862405d85d6 Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Tue, 17 Oct 2023 10:07:59 +0100 Subject: [PATCH 07/13] condense repeated elements into source policies: - write_metadata - create_write_lambda_logs rename create_write_lambda_logs for clarity formatting --- terraform/environments/data-platform/iam.tf | 257 ++++++------------ .../environments/data-platform/lambda.tf | 2 +- 2 files changed, 81 insertions(+), 178 deletions(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 9f4249e5f06..98b08dabe1e 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -18,20 +18,23 @@ data "aws_iam_policy_document" "log_to_bucket" { data "aws_iam_policy_document" "read_metadata" { statement { - sid = "s3ReadMetadata" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:ListBucket", - ] - resources = [ - "${module.metadata_s3_bucket.bucket.arn}", - "${module.metadata_s3_bucket.bucket.arn}/*" - ] + sid = "s3ReadMetadata" + effect = "Allow" + actions = ["s3:GetObject"] + resources = ["${module.metadata_s3_bucket.bucket.arn}/*"] } } -data "aws_iam_policy_document" "iam_policy_document_for_docs_lambda" { +data "aws_iam_policy_document" "write_metadata" { + statement { + sid = "s3WriteMetadata" + effect = "Allow" + actions = ["s3:PutObject"] + resources = ["${module.metadata_s3_bucket.bucket.arn}/*"] + } +} + +data "aws_iam_policy_document" "create_write_lambda_logs" { statement { sid = "LambdaLogGroup" effect = "Allow" @@ -41,29 +44,12 @@ data "aws_iam_policy_document" "iam_policy_document_for_docs_lambda" { } data "aws_iam_policy_document" "athena_load_lambda_function_policy" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.read_metadata.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] - statement { - sid = "AllowLambdaToCreateLogGroup" - effect = "Allow" - actions = [ - "logs:CreateLogGroup" - ] - resources = [ - format("arn:aws:logs:eu-west-2:%s:*", data.aws_caller_identity.current.account_id) - ] - } - statement { - sid = "AllowLambdaToWriteLogsToGroup" - effect = "Allow" - actions = [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - resources = [ - format("arn:aws:logs:eu-west-2:%s:*", data.aws_caller_identity.current.account_id) - ] - } statement { sid = "s3Access" effect = "Allow" @@ -135,29 +121,12 @@ data "aws_iam_policy_document" "athena_load_lambda_function_policy" { } data "aws_iam_policy_document" "landing_to_raw_lambda_policy" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.read_metadata.json, + data.data.aws_iam_policy_document.create_write_lambda_logs.json, + ] - statement { - sid = "AllowLambdaToCreateLogGroup" - effect = "Allow" - actions = [ - "logs:CreateLogGroup" - ] - resources = [ - format("arn:aws:logs:eu-west-2:%s:*", data.aws_caller_identity.current.account_id) - ] - } - statement { - sid = "AllowLambdaToWriteLogsToGroup" - effect = "Allow" - actions = [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - resources = [ - format("arn:aws:logs:eu-west-2:%s:*", data.aws_caller_identity.current.account_id) - ] - } statement { sid = "getLandingData" effect = "Allow" @@ -185,18 +154,17 @@ data "aws_iam_policy_document" "landing_to_raw_lambda_policy" { } data "aws_iam_policy_document" "iam_policy_document_for_authorizer_lambda" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json] - - statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] - } + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] } data "aws_iam_policy_document" "iam_policy_document_for_get_glue_metadata_lambda" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json] + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] statement { sid = "GlueReadOnly" effect = "Allow" @@ -207,21 +175,23 @@ data "aws_iam_policy_document" "iam_policy_document_for_get_glue_metadata_lambda "arn:aws:glue:${local.region}:${local.account_id}:table/*" ] } - statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] - } } data "aws_iam_policy_document" "iam_policy_document_for_presigned_url_lambda" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.read_metadata.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] statement { - sid = "GetPutDataObject" - effect = "Allow" - actions = ["s3:GetObject", "s3:PutObject", "s3:ListBucket"] + sid = "GetPutDataObject" + effect = "Allow" + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket", + ] resources = [ "${module.data_s3_bucket.bucket.arn}/raw/*", "${module.logs_s3_bucket.bucket.arn}/logs/*", @@ -229,13 +199,6 @@ data "aws_iam_policy_document" "iam_policy_document_for_presigned_url_lambda" { "${module.logs_s3_bucket.bucket.arn}/logs", ] } - - statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] - } } # API Gateway authoriser IAM permissions @@ -443,9 +406,15 @@ data "aws_iam_policy_document" "logs_s3_bucket_policy_document" { identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:user/cicd-member-user"] } - actions = ["s3:PutObject", "s3:ListBucket"] + actions = [ + "s3:PutObject", + "s3:ListBucket" + ] - resources = [module.logs_s3_bucket.bucket.arn, "${module.logs_s3_bucket.bucket.arn}/*"] + resources = [ + module.logs_s3_bucket.bucket.arn, + "${module.logs_s3_bucket.bucket.arn}/*", + ] } statement { @@ -473,16 +442,12 @@ data "aws_iam_policy_document" "logs_s3_bucket_policy_document" { # api gateway create data product metdata permissions data "aws_iam_policy_document" "iam_policy_document_for_create_metadata_lambda" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json] - - statement { - sid = "GetPutMetadata" - effect = "Allow" - actions = ["s3:GetObject", "s3:PutObject"] - resources = [ - "${module.metadata_s3_bucket.bucket.arn}/*" - ] - } + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.read_metadata.json, + data.aws_iam_policy_document.write_metadata.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] statement { sid = "ListBucket" @@ -492,32 +457,14 @@ data "aws_iam_policy_document" "iam_policy_document_for_create_metadata_lambda" module.metadata_s3_bucket.bucket.arn ] } - - statement { - sid = "AllowLambdaToCreateLogGroup" - effect = "Allow" - actions = [ - "logs:CreateLogGroup" - ] - resources = [ - format("arn:aws:logs:eu-west-2:%s:*", data.aws_caller_identity.current.account_id) - ] - } - statement { - sid = "AllowLambdaToWriteLogsToGroup" - effect = "Allow" - actions = [ - "logs:CreateLogStream", - "logs:PutLogEvents" - ] - resources = [ - format("arn:aws:logs:eu-west-2:%s:*", data.aws_caller_identity.current.account_id) - ] - } } data "aws_iam_policy_document" "iam_policy_document_for_reload_data_product_lambda" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.read_metadata.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] statement { sid = "ListBucket" @@ -542,36 +489,23 @@ data "aws_iam_policy_document" "iam_policy_document_for_reload_data_product_lamb "*" ] } - statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] - } } data "aws_iam_policy_document" "iam_policy_document_for_get_schema_lambda" { - statement { - sid = "s3LogAccess" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:PutObject", - ] - resources = [ - "${module.s3-bucket.bucket.arn}/logs/*" - ] - } - statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] - } + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.read_metadata.json, + data.aws_iam_policy_document.write_metadata.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] } data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files_lambda" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.read_metadata.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] statement { sid = "ListBucket" @@ -587,44 +521,13 @@ data "aws_iam_policy_document" "iam_policy_document_for_resync_unprocessed_files actions = ["lambda:InvokeFunction"] resources = [module.data_product_athena_load_lambda.lambda_function_arn] } - - statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] - } } data "aws_iam_policy_document" "iam_policy_document_for_create_schema_lambda" { - source_policy_documents = [data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json] - - statement { - sid = "s3MetadataAccessPlaceholder" - effect = "Allow" - actions = [ - "s3:GetObject", - "s3:ListBucket" - ] - resources = [ - "${module.s3-bucket.bucket.arn}/*", - "${module.s3-bucket.bucket.arn}" - ] - } - - statement { - sid = "s3MetadataWrite" - effect = "Allow" - actions = ["s3:PutObject"] - resources = [ - "${module.metadata_s3_bucket.bucket.arn}/*", - - ] - } - statement { - sid = "LambdaLogGroup" - effect = "Allow" - actions = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"] - resources = ["arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/*"] - } + source_policy_documents = [ + data.aws_iam_policy_document.log_to_bucket.json, + data.aws_iam_policy_document.read_metadata.json, + data.aws_iam_policy_document.write_metadata.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, + ] } diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index e6d6acd757e..54fc25e3eae 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -6,7 +6,7 @@ module "data_product_docs_lambda" { function_name = "data_product_docs_${local.environment}" role_name = "docs_lambda_role_${local.environment}" policy_json_attached = true - policy_json = data.aws_iam_policy_document.iam_policy_document_for_docs_lambda.json + policy_json = data.aws_iam_policy_document.create_write_lambda_logs.json create_role = true reserved_concurrent_executions = 1 From aa3c1ac678abb76d552c68c6cad8f2a60d9f0461 Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Tue, 17 Oct 2023 10:11:34 +0100 Subject: [PATCH 08/13] remove write access for get_schema lambda --- terraform/environments/data-platform/iam.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 98b08dabe1e..fa6ea9e60be 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -495,7 +495,6 @@ data "aws_iam_policy_document" "iam_policy_document_for_get_schema_lambda" { source_policy_documents = [ data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json, - data.aws_iam_policy_document.write_metadata.json, data.aws_iam_policy_document.create_write_lambda_logs.json, ] } From e01447943a9e1c9c9927afbd4a20b77121549742 Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Tue, 17 Oct 2023 10:29:07 +0100 Subject: [PATCH 09/13] fix typo --- terraform/environments/data-platform/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index fa6ea9e60be..7fe71ae4100 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -124,7 +124,7 @@ data "aws_iam_policy_document" "landing_to_raw_lambda_policy" { source_policy_documents = [ data.aws_iam_policy_document.log_to_bucket.json, data.aws_iam_policy_document.read_metadata.json, - data.data.aws_iam_policy_document.create_write_lambda_logs.json, + data.aws_iam_policy_document.create_write_lambda_logs.json, ] statement { From 1ddb888454772b079da53bc8bb68253e286d3f97 Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Tue, 17 Oct 2023 11:57:03 +0100 Subject: [PATCH 10/13] add ListBucket for read_metadata (all GetObjects incude ListBucket now) fix get_schema_lambda env vars (buckets) add required var to get_schema_lambda --- terraform/environments/data-platform/iam.tf | 11 +++++++---- terraform/environments/data-platform/lambda.tf | 9 ++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/terraform/environments/data-platform/iam.tf b/terraform/environments/data-platform/iam.tf index 7fe71ae4100..87d3e63228a 100644 --- a/terraform/environments/data-platform/iam.tf +++ b/terraform/environments/data-platform/iam.tf @@ -18,10 +18,13 @@ data "aws_iam_policy_document" "log_to_bucket" { data "aws_iam_policy_document" "read_metadata" { statement { - sid = "s3ReadMetadata" - effect = "Allow" - actions = ["s3:GetObject"] - resources = ["${module.metadata_s3_bucket.bucket.arn}/*"] + sid = "s3ReadMetadata" + effect = "Allow" + actions = ["s3:GetObject", "s3:ListBucket"] + resources = [ + "${module.metadata_s3_bucket.bucket.arn}/*", + "${module.metadata_s3_bucket.bucket.arn}", + ] } } diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index 54fc25e3eae..3dfbe07d641 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -302,6 +302,7 @@ module "get_schema_lambda" { description = "Fetch the schema for a table from S3" role_name = "get_schema_role_${local.environment}" policy_json = data.aws_iam_policy_document.iam_policy_document_for_get_schema_lambda.json + policy_json_attached = true function_name = "get_schema_${local.environment}" create_role = true reserved_concurrent_executions = 1 @@ -311,12 +312,6 @@ module "get_schema_lambda" { tracing_mode = "Active" memory_size = 512 - environment_variables = { - LOG_BUCKET = module.s3-bucket.bucket.id - METADATA_BUCKET = module.s3-bucket.bucket.id - RAW_DATA_BUCKET = module.s3-bucket.bucket.id - CURATED_DATA_BUCKET = module.s3-bucket.bucket.id - LANDING_ZONE_BUCKET = module.s3-bucket.bucket.id - } + environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) } From 80ee807476aa4282cc355c05a1afdca9c12812ce Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Tue, 17 Oct 2023 12:58:59 +0100 Subject: [PATCH 11/13] add moved block for renaming schema resource --- terraform/environments/data-platform/api.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/terraform/environments/data-platform/api.tf b/terraform/environments/data-platform/api.tf index b93abd60783..c51626aaa4e 100644 --- a/terraform/environments/data-platform/api.tf +++ b/terraform/environments/data-platform/api.tf @@ -132,6 +132,11 @@ resource "aws_api_gateway_resource" "schema_for_data_product_table_name" { rest_api_id = aws_api_gateway_rest_api.data_platform.id } +moved { + from = aws_api_gateway_resource.create_schema_for_data_product_table_name + to = aws_api_gateway_resource.schema_for_data_product_table_name +} + # /data-product/{data-product-name}/table/{table-name}/upload POST method resource "aws_api_gateway_method" "upload_data_for_data_product_table_name" { authorization = "CUSTOM" From 826a9e5cd3d60f8a7d652add494f84aacbe280bc Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Tue, 17 Oct 2023 13:52:58 +0100 Subject: [PATCH 12/13] add execution from api gateway to get_schema lambda --- terraform/environments/data-platform/lambda.tf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index 3dfbe07d641..2175c313c63 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -1,3 +1,12 @@ +locals { + AllowExecutionFromAPIGateway = { + action = "lambda:InvokeFunction" + function_name = "data_product_create_metadata_${local.environment}" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" + } +} + module "data_product_docs_lambda" { source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 application_name = "data_product_docs" @@ -310,8 +319,9 @@ module "get_schema_lambda" { image_uri = "374269020027.dkr.ecr.eu-west-2.amazonaws.com/data-platform-get-schema-lambda-ecr-repo:${local.get_schema_version}" timeout = 600 tracing_mode = "Active" - memory_size = 512 + memory_size = 128 environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) + allowed_triggers = local.AllowExecutionFromAPIGateway } From 1b1b46622f4262d6c555867827e8382fee1860be Mon Sep 17 00:00:00 2001 From: Tom Webber Date: Tue, 17 Oct 2023 14:08:13 +0100 Subject: [PATCH 13/13] fix allowed_triggers for get_schema remove unneeded key for other lambdas --- .../environments/data-platform/lambda.tf | 75 ++++++++----------- 1 file changed, 32 insertions(+), 43 deletions(-) diff --git a/terraform/environments/data-platform/lambda.tf b/terraform/environments/data-platform/lambda.tf index 2175c313c63..fbdf8fb0dfc 100644 --- a/terraform/environments/data-platform/lambda.tf +++ b/terraform/environments/data-platform/lambda.tf @@ -1,12 +1,3 @@ -locals { - AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - function_name = "data_product_create_metadata_${local.environment}" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" - } -} - module "data_product_docs_lambda" { source = "github.com/ministryofjustice/modernisation-platform-terraform-lambda-function?ref=a4392c1" # ref for V2.1 application_name = "data_product_docs" @@ -27,10 +18,9 @@ module "data_product_docs_lambda" { allowed_triggers = { AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - function_name = "data_product_docs_${local.environment}" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/*" + action = "lambda:InvokeFunction" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/*" } } @@ -61,10 +51,9 @@ module "data_product_authorizer_lambda" { allowed_triggers = { AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - function_name = "data_product_authorizer_${local.environment}" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/*" + action = "lambda:InvokeFunction" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/*" } } @@ -90,10 +79,9 @@ module "data_product_get_glue_metadata_lambda" { allowed_triggers = { AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - function_name = "data_product_get_glue_metadata_${local.environment}" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.get_glue_metadata.http_method}${aws_api_gateway_resource.get_glue_metadata.path}" + action = "lambda:InvokeFunction" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.get_glue_metadata.http_method}${aws_api_gateway_resource.get_glue_metadata.path}" } } @@ -121,10 +109,9 @@ module "data_product_landing_to_raw_lambda" { allowed_triggers = { AllowExecutionFromCloudWatch = { - action = "lambda:InvokeFunction" - function_name = "data_product_landing_to_raw_${local.environment}" - principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.object_created_data_landing.arn + action = "lambda:InvokeFunction" + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.object_created_data_landing.arn } } @@ -152,10 +139,9 @@ module "data_product_presigned_url_lambda" { allowed_triggers = { AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - function_name = "data_product_presigned_url_${local.environment}" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.upload_data_for_data_product_table_name.http_method}${aws_api_gateway_resource.upload_data_for_data_product_table_name.path}" + action = "lambda:InvokeFunction" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.upload_data_for_data_product_table_name.http_method}${aws_api_gateway_resource.upload_data_for_data_product_table_name.path}" } } @@ -185,10 +171,9 @@ module "data_product_athena_load_lambda" { allowed_triggers = { AllowExecutionFromCloudWatch = { - action = "lambda:InvokeFunction" - function_name = "data_product_athena_load_${local.environment}" - principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.object_created_raw_data.arn + action = "lambda:InvokeFunction" + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.object_created_raw_data.arn } } @@ -219,10 +204,9 @@ module "data_product_create_metadata_lambda" { allowed_triggers = { AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - function_name = "data_product_create_metadata_${local.environment}" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.register_data_product.http_method}${aws_api_gateway_resource.register_data_product.path}" + action = "lambda:InvokeFunction" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.register_data_product.http_method}${aws_api_gateway_resource.register_data_product.path}" } } @@ -295,13 +279,11 @@ module "data_product_create_schema_lambda" { allowed_triggers = { AllowExecutionFromAPIGateway = { - action = "lambda:InvokeFunction" - function_name = "data_product_create_metadata_${local.environment}" - principal = "apigateway.amazonaws.com" - source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" + action = "lambda:InvokeFunction" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.create_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" } } - } module "get_schema_lambda" { @@ -323,5 +305,12 @@ module "get_schema_lambda" { environment_variables = merge(local.logger_environment_vars, local.storage_environment_vars) - allowed_triggers = local.AllowExecutionFromAPIGateway + allowed_triggers = { + + AllowExecutionFromAPIGateway = { + action = "lambda:InvokeFunction" + principal = "apigateway.amazonaws.com" + source_arn = "arn:aws:execute-api:${local.region}:${local.account_id}:${aws_api_gateway_rest_api.data_platform.id}/*/${aws_api_gateway_method.get_schema_for_data_product_table_name.http_method}${aws_api_gateway_resource.schema_for_data_product_table_name.path}" + } + } }