diff --git a/.gitignore b/.gitignore index d2ab1428d5..40b0a6a6e8 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,12 @@ venv_aws/ # Backend coverage generated files htmlcov/ .coverage + +# Terraform +.terraform +*.tfstate +*.tfstate.backup +.terraform.lock.hcl + +# Terragrunt +.terragrunt-cache diff --git a/scripts/aws/lambda/TM-Extractor/README.md b/scripts/aws/lambda/TM-Extractor/README.md new file mode 100644 index 0000000000..10315c0239 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/README.md @@ -0,0 +1,61 @@ + +## TM-Extractor with Terragrunt + +This directory contains infrastructure config for deploying [tm-extractor](https://github.com/hotosm/TM-Extractor) on AWS Lambda. + +Check [tm-extractor](https://github.com/hotosm/TM-Extractor) for resources created and exporter script. + +To get it running on your environment follow the following. + +### Prerequisites + +This are list of things you will need. +* AWS IAM User with permission for creating resources such as lambda function, cloudwatch group, cloudwatch event etc. +See all resource [here](https://github.com/hotosm/TM-Extractor/blob/ec37c1318325c534b4ac47f057263050e6e92f03/main.tf). +* [Terraform](https://developer.hashicorp.com/terraform/tutorials/aws-get-started/install-cli) or (OpenTofu)[https://opentofu.org/docs/intro/install/] +* [Terragrunt](https://terragrunt.gruntwork.io/docs/getting-started/install/#install-terragrunt) +> **_NOTE:_** Please check compatibility note. https://terragrunt.gruntwork.io/docs/getting-started/supported-versions/ +Tested on: Terraform v1.5.5 & terragrunt version v0.54.9. + +### Environments + +Before running deployment, make sure you have the following environment variables exported. +- `TF_VAR_rawdata_api_auth_token`, Auth token for raw data api, Request [here](https://github.com/hotosm/raw-data-api/). +- `TF_VAR_raw_data_api`, API endpoint. Defaults to https://api-prod.raw-data.hotosm.org/v1 +- `TF_VAR_active_projects_api_base_url`, Your [tasking-manager](https://github.com/hotosm/tasking-manager) instance. +Defaults to https://tasking-manager-staging-api.hotosm.org/api/v2 + +## Inputs and Local Variables +`env.hcl` within each deployment environment overrides the input vars. +``` +locals { + environment = "stag" + # Appends to all the resource names eg: tm-extrator-hotosm-lambda-stag + + account_name = "hotosm" + # Also appends to the resource names for better naming + + aws_profile = "default" + # Acts as AWS_PROFILE environment var, If you have multiple profile defined in ~/.aws/credentials, you can set these so each envionment deployment uses its own set of profiles. + + aws_region = "ap-south-1" + # The AWS region the resources should be created on. + + application = "tasking-manager" + team = "HOTOSM" + creator = "HOTOSM" + owner = "HOTOSM" + # Above are used for generating tags for AWS resources. +} +``` + +## Plan and Apply +- `cd` into `scripts/aws/lambda/TM-Extractor//tm-extractor`. +- `terragrunt init`, initializes providers and resources. +- `terragrunt plan`, run a plan to check for infrastructure changes. +- `terragrunt apply`, applies the configuration. + + +_For more information, please refer to the [TM-Extractor](https://github.com/hotosm/TM-Extractor)_ repository. + +

(back to top)

\ No newline at end of file diff --git a/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl new file mode 100644 index 0000000000..6f477a8855 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/_envcommon/lambda.hcl @@ -0,0 +1,48 @@ +# --------------------------------------------------------------------------------------------------------------------- +# COMMON TERRAGRUNT CONFIGURATION +# This is the common component configuration for mysql. The common variables for each environment to +# deploy mysql are defined here. This configuration will be merged into the environment configuration +# via an include block. +# --------------------------------------------------------------------------------------------------------------------- + +# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the +# working directory, into a temporary folder, and execute your Terraform commands in that folder. If any environment +# needs to deploy a different module version, it should redefine this block with a different ref to override the +# deployed version. +terraform { + # Sourcing from https://github.com/hotosm/TM-Extractor/ + source = "${local.base_source_url}?ref=v1.0.1" +} + +# --------------------------------------------------------------------------------------------------------------------- +# Locals are named constants that are reusable within the configuration. +# --------------------------------------------------------------------------------------------------------------------- +locals { + # Automatically load environment-level variables + environment_vars = read_terragrunt_config(find_in_parent_folders("env.hcl")) + + # Extract out common variables for reuse + env = local.environment_vars.locals.environment + + # Expose the base source URL so different versions of the module can be deployed in different environments. This will + # be used to construct the terraform block in the child terragrunt configurations. + base_source_url = "git::https://github.com/hotosm/TM-Extractor/" +} + +# --------------------------------------------------------------------------------------------------------------------- +# MODULE PARAMETERS +# These are the variables we have to pass in to use the module. This defines the parameters that are common across all +# environments. +# --------------------------------------------------------------------------------------------------------------------- +# Defaults, overridden by env.hcl + +inputs = { + iam_lambda_role_name = "tm-extractor-role" + cw_retention_in_days = "14" + lambda_environment = "dev" + lambda_memory_size = "128" + lambda_timeout = "14" + lambda_cron_expression = "cron(0 0 * * ? *)" + raw_data_api = "https://api-prod.raw-data.hotosm.org/v1" + config_json = "config.json" +} diff --git a/scripts/aws/lambda/TM-Extractor/dev/env.hcl b/scripts/aws/lambda/TM-Extractor/dev/env.hcl new file mode 100644 index 0000000000..5d5d8c4cab --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/dev/env.hcl @@ -0,0 +1,14 @@ +# Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to +# feed forward to the child modules. +# Check ../README.md ## Inputs and Local Variables for details. + +locals { + environment = "dev" + account_name = "naxadevelopers" + aws_profile = "default" + aws_region = "ap-south-1" + application = "tasking-manager" + team = "HOTOSM" + creator = "HOTOSM" + owner = "HOTOSM" +} \ No newline at end of file diff --git a/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl new file mode 100644 index 0000000000..ddb6710a01 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/dev/tm-extractor/terragrunt.hcl @@ -0,0 +1,32 @@ +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# This is the configuration for Terragrunt, a thin wrapper for Terraform that helps keep your code DRY and +# maintainable: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# Include configurations that are common used across multiple environments. +# --------------------------------------------------------------------------------------------------------------------- + +# Include the root `terragrunt.hcl` configuration. The root configuration contains settings that are common across all +# components and environments, such as how to configure remote state. +include "root" { + path = find_in_parent_folders() +} + +# Include the envcommon configuration for the component. The envcommon configuration contains settings that are common +# for the component across all environments. +include "envcommon" { + path = "${dirname(find_in_parent_folders())}/_envcommon/lambda.hcl" +} + +# --------------------------------------------------------------------------------------------------------------------- +# Override parameters for this environment +# --------------------------------------------------------------------------------------------------------------------- + +# For production, we want to specify bigger instance classes and storage, so we specify override parameters here. These +# inputs get merged with the common inputs from the root and the envcommon lambda.hcl +inputs = { + zip_output_dir = "files" + active_projects_api_base_url = "https://tm.naxa.com.np/api/v2" +} diff --git a/scripts/aws/lambda/TM-Extractor/prod/env.hcl b/scripts/aws/lambda/TM-Extractor/prod/env.hcl new file mode 100644 index 0000000000..f36d715ca1 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/prod/env.hcl @@ -0,0 +1,14 @@ +# Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to +# feed forward to the child modules. +# Check ../README.md ## Inputs and Local Variables for details. + +locals { + environment = "prod" + account_name = "hotosm" + aws_profile = "default" + aws_region = "ap-south-1" + application = "tasking-manager" + team = "HOTOSM" + creator = "HOTOSM" + owner = "HOTOSM" +} \ No newline at end of file diff --git a/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl new file mode 100644 index 0000000000..a189413910 --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/prod/tm-extractor/terragrunt.hcl @@ -0,0 +1,31 @@ +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# This is the configuration for Terragrunt, a thin wrapper for Terraform that helps keep your code DRY and +# maintainable: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# Include configurations that are common used across multiple environments. +# --------------------------------------------------------------------------------------------------------------------- + +# Include the root `terragrunt.hcl` configuration. The root configuration contains settings that are common across all +# components and environments, such as how to configure remote state. +include "root" { + path = find_in_parent_folders() +} + +# Include the envcommon configuration for the component. The envcommon configuration contains settings that are common +# for the component across all environments. +include "envcommon" { + path = "${dirname(find_in_parent_folders())}/_envcommon/lambda.hcl" +} + +# --------------------------------------------------------------------------------------------------------------------- +# Override parameters for this environment +# --------------------------------------------------------------------------------------------------------------------- + +# For production, we want to specify bigger instance classes and storage, so we specify override parameters here. These +# inputs get merged with the common inputs from the root and the envcommon lambda.hcl +inputs = { + active_projects_api_base_url = "https://tasking-manager-tm4-production-api.hotosm.org/api/v2" +} diff --git a/scripts/aws/lambda/TM-Extractor/stag/env.hcl b/scripts/aws/lambda/TM-Extractor/stag/env.hcl new file mode 100644 index 0000000000..bf584eb7ed --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/stag/env.hcl @@ -0,0 +1,14 @@ +# Set common variables for the environment. This is automatically pulled in in the root terragrunt.hcl configuration to +# feed forward to the child modules. +# Check ../README.md ## Inputs and Local Variables for details. + +locals { + environment = "stag" + account_name = "hotosm" + aws_profile = "default" + aws_region = "ap-south-1" + application = "tasking-manager" + team = "HOTOSM" + creator = "HOTOSM" + owner = "HOTOSM" +} \ No newline at end of file diff --git a/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl new file mode 100644 index 0000000000..567fc8049b --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/stag/tm-extractor/terragrunt.hcl @@ -0,0 +1,32 @@ +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# This is the configuration for Terragrunt, a thin wrapper for Terraform that helps keep your code DRY and +# maintainable: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- + +# --------------------------------------------------------------------------------------------------------------------- +# Include configurations that are common used across multiple environments. +# --------------------------------------------------------------------------------------------------------------------- + +# Include the root `terragrunt.hcl` configuration. The root configuration contains settings that are common across all +# components and environments, such as how to configure remote state. +include "root" { + path = find_in_parent_folders() +} + +# Include the envcommon configuration for the component. The envcommon configuration contains settings that are common +# for the component across all environments. +include "envcommon" { + path = "${dirname(find_in_parent_folders())}/_envcommon/lambda.hcl" +} + +# --------------------------------------------------------------------------------------------------------------------- +# Override parameters for this environment +# --------------------------------------------------------------------------------------------------------------------- + +# For production, we want to specify bigger instance classes and storage, so we specify override parameters here. These +# inputs get merged with the common inputs from the root and the envcommon lambda.hcl +inputs = { + zip_output_dir = "files" + active_projects_api_base_url = "https://tasking-manager-staging-api.hotosm.org/api/v2" +} diff --git a/scripts/aws/lambda/TM-Extractor/terragrunt.hcl b/scripts/aws/lambda/TM-Extractor/terragrunt.hcl new file mode 100644 index 0000000000..fca61f78ac --- /dev/null +++ b/scripts/aws/lambda/TM-Extractor/terragrunt.hcl @@ -0,0 +1,89 @@ +# --------------------------------------------------------------------------------------------------------------------- +# TERRAGRUNT CONFIGURATION +# Terragrunt is a thin wrapper for Terraform that provides extra tools for working with multiple Terraform modules, +# remote state, and locking: https://github.com/gruntwork-io/terragrunt +# --------------------------------------------------------------------------------------------------------------------- + +locals { + # Automatically load environment-level variables + environment_vars = read_terragrunt_config(find_in_parent_folders("env.hcl")) + + # Extract the variables we need for easy access + account_name = local.environment_vars.locals.account_name + aws_profile = local.environment_vars.locals.aws_profile + aws_region = local.environment_vars.locals.aws_region + environment = local.environment_vars.locals.environment + application = local.environment_vars.locals.application + team = local.environment_vars.locals.team + creator = local.environment_vars.locals.creator + owner = local.environment_vars.locals.owner +} + +# Generate an AWS provider block +generate "provider" { + path = "provider.tf" + if_exists = "overwrite_terragrunt" + contents = <