diff --git a/.gitignore b/.gitignore index d2ab1428d5..14c7f30aec 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,17 @@ venv_aws/ # Backend coverage generated files htmlcov/ .coverage + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +crash.*.log + +#To be generated by terraform in build time +*cron_raw_data.zip* diff --git a/scripts/aws/terraform/export-tool-lambda-cron/.terraform.lock.hcl b/scripts/aws/terraform/export-tool-lambda-cron/.terraform.lock.hcl new file mode 100644 index 0000000000..d4b0d084f0 --- /dev/null +++ b/scripts/aws/terraform/export-tool-lambda-cron/.terraform.lock.hcl @@ -0,0 +1,44 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/archive" { + version = "2.4.0" + hashes = [ + "h1:cJokkjeH1jfpG4QEHdRx0t2j8rr52H33A7C/oX73Ok4=", + "zh:18e408596dd53048f7fc8229098d0e3ad940b92036a24287eff63e2caec72594", + "zh:392d4216ecd1a1fd933d23f4486b642a8480f934c13e2cae3c13b6b6a7e34a7b", + "zh:655dd1fa5ca753a4ace21d0de3792d96fff429445717f2ce31c125d19c38f3ff", + "zh:70dae36c176aa2b258331ad366a471176417a94dd3b4985a911b8be9ff842b00", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:7d8c8e3925f1e21daf73f85983894fbe8868e326910e6df3720265bc657b9c9c", + "zh:a032ec0f0aee27a789726e348e8ad20778c3a1c9190ef25e7cff602c8d175f44", + "zh:b8e50de62ba185745b0fe9713755079ad0e9f7ac8638d204de6762cc36870410", + "zh:c8ad0c7697a3d444df21ff97f3473a8604c8639be64afe3f31b8ec7ad7571e18", + "zh:df736c5a2a7c3a82c5493665f659437a22f0baf8c2d157e45f4dd7ca40e739fc", + "zh:e8ffbf578a0977074f6d08aa8734e36c726e53dc79894cfc4f25fadc4f45f1df", + "zh:efea57ff23b141551f92b2699024d356c7ffd1a4ad62931da7ed7a386aef7f1f", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.67.0" + constraints = "~> 4.67.0" + hashes = [ + "h1:5Zfo3GfRSWBaXs4TGQNOflr1XaYj6pRnVJLX5VAjFX4=", + "zh:0843017ecc24385f2b45f2c5fce79dc25b258e50d516877b3affee3bef34f060", + "zh:19876066cfa60de91834ec569a6448dab8c2518b8a71b5ca870b2444febddac6", + "zh:24995686b2ad88c1ffaa242e36eee791fc6070e6144f418048c4ce24d0ba5183", + "zh:4a002990b9f4d6d225d82cb2fb8805789ffef791999ee5d9cb1fef579aeff8f1", + "zh:559a2b5ace06b878c6de3ecf19b94fbae3512562f7a51e930674b16c2f606e29", + "zh:6a07da13b86b9753b95d4d8218f6dae874cf34699bca1470d6effbb4dee7f4b7", + "zh:768b3bfd126c3b77dc975c7c0e5db3207e4f9997cf41aa3385c63206242ba043", + "zh:7be5177e698d4b547083cc738b977742d70ed68487ce6f49ecd0c94dbf9d1362", + "zh:8b562a818915fb0d85959257095251a05c76f3467caa3ba95c583ba5fe043f9b", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9c385d03a958b54e2afd5279cd8c7cbdd2d6ca5c7d6a333e61092331f38af7cf", + "zh:b3ca45f2821a89af417787df8289cb4314b273d29555ad3b2a5ab98bb4816b3b", + "zh:da3c317f1db2469615ab40aa6baba63b5643bae7110ff855277a1fb9d8eb4f2c", + "zh:dc6430622a8dc5cdab359a8704aec81d3825ea1d305bbb3bbd032b1c6adfae0c", + "zh:fac0d2ddeadf9ec53da87922f666e1e73a603a611c57bcbc4b86ac2821619b1d", + ] +} diff --git a/scripts/aws/terraform/export-tool-lambda-cron/backend.tf b/scripts/aws/terraform/export-tool-lambda-cron/backend.tf new file mode 100644 index 0000000000..a63bf44f57 --- /dev/null +++ b/scripts/aws/terraform/export-tool-lambda-cron/backend.tf @@ -0,0 +1,10 @@ +## Initializes Tf backend to use S3, create s3 bucket first. + +# terraform { +# backend "s3" { +# bucket = "hotosm-tm-terraform-statefiles" +# dynamodb_table = "hotosm-tm-terraform-locks" +# key = "terraform.state" +# region = "ap-south-1" +# } +# } \ No newline at end of file diff --git a/scripts/aws/terraform/export-tool-lambda-cron/lambda_function.py b/scripts/aws/terraform/export-tool-lambda-cron/lambda_function.py new file mode 100644 index 0000000000..634a7d902d --- /dev/null +++ b/scripts/aws/terraform/export-tool-lambda-cron/lambda_function.py @@ -0,0 +1,93 @@ +import json +import requests +import os + +mapping_types_reverse = { + 1 : ["highway", "roads"], + 2 : ["building", "buildings"], + 3 : ["waterway", "waterways"], + 4 : ["landuse", "landUse"], + 5 : ["other", "other"] +} + +output_types = ["geojson", "shp", "kml"] + +raw_data_api = os.environ.get('RAW_DATA_API', "https://api-prod.raw-data.hotosm.org/v1/snapshot/") +rawdata_api_auth_token = os.environ.get('RAWDATA_API_AUTH_TOKEN', "") +active_projects_api_base_url = os.environ.get('ACTIVE_PROJECTS_API_BASE_URL', "") + +headers = { + "Content-Type": "application/json", + "Access-Token": rawdata_api_auth_token +} + +def generate_payload(project_id: int, mapping_type: str, output_type: str, bbox_geometry: str) -> dict: + """ + Generate a payload data dictionary for the given project, mapping type, output type, and bounding box geometry. + + Args: + project_id (int): The ID of the project. + mapping_type (str): The type of mapping. + output_type (str): The type of output. + bbox_geometry (str): The bounding box geometry. + + Returns: + dict: The payload data dictionary. + """ + payload_data = { + "bindZip": "true", + "centroid": "false", + "fileName": f"hotosm_project_{project_id}_{mapping_type[1]}", + "outputType": output_type, + "uuid": "false", + "useStWithin": "true", + "filters": { + "tags": { + "all_geometry": { + "join_or": { + mapping_type[0]: [], + } + } + }, + "attributes": { + "all_geometry": [ + "name", + "" + ] + } + }, + "geometry": bbox_geometry + } + return payload_data + +def lambda_handler(event, context): + """ + This function retrieves active projects from Tasking Manager API and generates payloads for each project. + The payloads are then used to call the Raw Data API. + """ + + time_interval = 24 + active_projects_api = f"{active_projects_api_base_url}/api/v2/projects/queries/active/?interval={time_interval}" + active_projects_api_response = requests.get(active_projects_api) + + if active_projects_api_response.status_code == 200: + active_projects = active_projects_api_response.json() + + for feature in active_projects['features']: + geometry = feature['geometry'] + project_id = feature['properties'].get('project_id') + mapping_types = feature['properties'].get('mapping_types') + + if mapping_types is not None: + for mapping_type in mapping_types: + for output_type in output_types: + mapping_type_value = mapping_types_reverse.get(mapping_type, None) + if mapping_type_value is not None: + payload = generate_payload(project_id, mapping_type_value, output_type, geometry) + payload_json = json.dumps(payload) + response = requests.post(raw_data_api, headers=headers, data=payload_json) + print(response.json()) + +# Commented for Lambda use, For CLI Use Uncomment. +if __name__ == "__main__": + lambda_handler(event, context) \ No newline at end of file diff --git a/scripts/aws/terraform/export-tool-lambda-cron/lambda_raw_data_cron_layer.zip b/scripts/aws/terraform/export-tool-lambda-cron/lambda_raw_data_cron_layer.zip new file mode 100644 index 0000000000..5d506ed7ff Binary files /dev/null and b/scripts/aws/terraform/export-tool-lambda-cron/lambda_raw_data_cron_layer.zip differ diff --git a/scripts/aws/terraform/export-tool-lambda-cron/main.tf b/scripts/aws/terraform/export-tool-lambda-cron/main.tf new file mode 100644 index 0000000000..efb419dc24 --- /dev/null +++ b/scripts/aws/terraform/export-tool-lambda-cron/main.tf @@ -0,0 +1,144 @@ + +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["lambda.amazonaws.com"] + } + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "iam_for_lambda_tm" { + name = "iam_for_lambda_tm" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +data "archive_file" "lambda" { + type = "zip" + source_file = "lambda_function.py" + output_path = "cron_raw_data.zip" +} + +resource "aws_lambda_layer_version" "lambda_layer" { + filename = "lambda_raw_data_cron_layer.zip" + layer_name = "lambda_raw_data_cron_layer" + + compatible_runtimes = ["python3.8"] +} +resource "aws_cloudwatch_log_group" "lambda_raw_data_cron" { + name = "/aws/lambda/${aws_lambda_function.lambda_raw_data.function_name}" + retention_in_days = 14 + lifecycle { + prevent_destroy = false + } + tags = { + Application = "lambda" + } +} + +resource "aws_lambda_function" "lambda_raw_data" { + + filename = "cron_raw_data.zip" + function_name = "lambda_raw_data_cron" + role = aws_iam_role.iam_for_lambda_tm.arn + handler = "lambda_function.lambda_handler" + memory_size = 128 + timeout = 20 # To be Increased if active projects are more. + layers = [ aws_lambda_layer_version.lambda_layer.id ] + + source_code_hash = data.archive_file.lambda.output_base64sha256 + + runtime = "python3.9" + # To be accessed from Environmnet varible TF_VAR_rawdata_api_auth_token & TF_VAR_active_projects_api_base_url. + depends_on = [ + aws_lambda_layer_version.lambda_layer, + aws_iam_role.iam_for_lambda_tm, + aws_iam_role_policy_attachment.lambda_logs, + ] + environment { + variables = { + ACTIVE_PROJECTS_API_BASE_URL = "${var.active_projects_api_base_url}", + RAWDATA_API_AUTH_TOKEN = "${var.rawdata_api_auth_token}" + } + } + + tracing_config { + mode = "PassThrough" + } + lifecycle { + ignore_changes = [layers] # Ignore changes to layers for now + } +} + +resource "aws_iam_policy" "lambda_logging" { + name = "lambda_raw_data_logging" + path = "/" + description = "IAM policy for logging from this lambda function" + + policy = <