From 377f984ed754a8b12dbc8d3de800b9815e48c2af Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Thu, 28 Apr 2022 11:21:55 +0200 Subject: [PATCH 1/5] feat: Added terraform_wrapper_module_for_each hook --- .github/.container-structure-test-config.yaml | 5 + .pre-commit-hooks.yaml | 11 + Dockerfile | 15 +- README.md | 39 +- hooks/terraform_wrapper_module_for_each.sh | 409 ++++++++++++++++++ 5 files changed, 471 insertions(+), 8 deletions(-) create mode 100755 hooks/terraform_wrapper_module_for_each.sh diff --git a/.github/.container-structure-test-config.yaml b/.github/.container-structure-test-config.yaml index 13880816f..a860febe4 100644 --- a/.github/.container-structure-test-config.yaml +++ b/.github/.container-structure-test-config.yaml @@ -55,6 +55,11 @@ commandTests: args: [ "--version" ] expectedOutput: [ "([0-9]+\\.){2}[0-9]+\\n$" ] + - name: "hcledit" + command: "hcledit" + args: [ "version" ] + expectedOutput: [ "([0-9]+\\.){2}[0-9]+\\n$" ] + fileExistenceTests: - name: 'terrascan init' path: '/root/.terrascan/pkg/policies/opa/rego/github/github_repository/privateRepoEnabled.rego' diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index dbee5e2a7..4f554ea4d 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -113,6 +113,17 @@ exclude: \.terraform\/.*$ require_serial: true +- id: terraform_wrapper_module_for_each + name: Terraform wrapper with for_each in module + description: Generate Terraform wrappers with for_each in module. + entry: hooks/terraform_wrapper_module_for_each.sh + language: script + pass_filenames: false + always_run: false + require_serial: true + files: \.tf$ + exclude: \.terraform\/.*$ + - id: terrascan name: terrascan description: Runs terrascan on Terraform templates. diff --git a/Dockerfile b/Dockerfile index 434c07ba5..e592a340c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,6 +35,7 @@ ARG TERRASCAN_VERSION=${TERRASCAN_VERSION:-false} ARG TFLINT_VERSION=${TFLINT_VERSION:-false} ARG TFSEC_VERSION=${TFSEC_VERSION:-false} ARG TFUPDATE_VERSION=${TFUPDATE_VERSION:-false} +ARG HCLEDIT_VERSION=${HCLEDIT_VERSION:-false} # Tricky thing to install all tools by set only one arg. @@ -49,7 +50,8 @@ RUN if [ "$INSTALL_ALL" != "false" ]; then \ echo "export TERRASCAN_VERSION=latest" >> /.env && \ echo "export TFLINT_VERSION=latest" >> /.env && \ echo "export TFSEC_VERSION=latest" >> /.env && \ - echo "export TFUPDATE_VERSION=latest" >> /.env \ + echo "export TFUPDATE_VERSION=latest" >> /.env && \ + echo "export HCLEDIT_VERSION=latest" >> /.env \ ; else \ touch /.env \ ; fi @@ -138,6 +140,16 @@ RUN . /.env && \ ) && tar -xzf tfupdate.tgz tfupdate && rm tfupdate.tgz \ ; fi +# hcledit +RUN . /.env && \ + if [ "$HCLEDIT_VERSION" != "false" ]; then \ + ( \ + HCLEDIT_RELEASES="https://api.github.com/repos/minamijoyo/hcledit/releases" && \ + [ "$HCLEDIT_VERSION" = "latest" ] && curl -L "$(curl -s ${HCLEDIT_RELEASES}/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > hcledit.tgz \ + || curl -L "$(curl -s ${HCLEDIT_RELEASES} | grep -o -E -m 1 "https://.+?${HCLEDIT_VERSION}_linux_amd64.tar.gz")" > hcledit.tgz \ + ) && tar -xzf hcledit.tgz hcledit && rm hcledit.tgz \ + ; fi + # Checking binaries versions and write it to debug file RUN . /.env && \ F=tools_versions_info && \ @@ -151,6 +163,7 @@ RUN . /.env && \ (if [ "$TFLINT_VERSION" != "false" ]; then ./tflint --version >> $F; else echo "tflint SKIPPED" >> $F ; fi) && \ (if [ "$TFSEC_VERSION" != "false" ]; then echo "tfsec $(./tfsec --version)" >> $F; else echo "tfsec SKIPPED" >> $F ; fi) && \ (if [ "$TFUPDATE_VERSION" != "false" ]; then echo "tfupdate $(./tfupdate --version)" >> $F; else echo "tfupdate SKIPPED" >> $F ; fi) && \ + (if [ "$HCLEDIT_VERSION" != "false" ]; then echo "hcledit $(./hcledit version)" >> $F; else echo "hcledit SKIPPED" >> $F ; fi) && \ echo -e "\n\n" && cat $F && echo -e "\n\n" diff --git a/README.md b/README.md index 4fe512c5f..91b6488fb 100644 --- a/README.md +++ b/README.md @@ -45,11 +45,12 @@ If you are using `pre-commit-terraform` already or want to support its developme * [terraform_tflint](#terraform_tflint) * [terraform_tfsec](#terraform_tfsec) * [terraform_validate](#terraform_validate) + * [terraform_wrapper_module_for_each](#terraform_wrapper_module_for_each) * [terrascan](#terrascan) * [tfupdate](#tfupdate) * [Authors](#authors) * [License](#license) - * [Additional terms of use for users from Russia and Belarus](#additional-terms-of-use-for-users-from-russia-and-belarus) + * [Additional information for users from Russia and Belarus](#additional-information-for-users-from-russia-and-belarus) ## How to install @@ -67,6 +68,7 @@ If you are using `pre-commit-terraform` already or want to support its developme * [`infracost`](https://github.com/infracost/infracost) required for `infracost_breakdown` hook. * [`jq`](https://github.com/stedolan/jq) required for `infracost_breakdown` hook. * [`tfupdate`](https://github.com/minamijoyo/tfupdate) required for `tfupdate` hook. +* [`hcledit`](https://github.com/minamijoyo/hcledit) required for `terraform_wrapper_module_for_each` hook.
Docker
@@ -104,6 +106,7 @@ docker build -t pre-commit-terraform \ --build-arg TFLINT_VERSION=0.31.0 \ --build-arg TFSEC_VERSION=latest \ --build-arg TFUPDATE_VERSION=latest \ + --build-arg HCLEDIT_VERSION=latest \ . ``` @@ -115,7 +118,7 @@ Set `-e PRE_COMMIT_COLOR=never` to disable the color output in `pre-commit`.
MacOS
```bash -brew install pre-commit terraform-docs tflint tfsec checkov terrascan infracost tfupdate jq +brew install pre-commit terraform-docs tflint tfsec checkov terrascan infracost tfupdate hcledit jq ```
@@ -137,6 +140,7 @@ curl -L "$(curl -s https://api.github.com/repos/accurics/terrascan/releases/late sudo apt install -y jq && \ curl -L "$(curl -s https://api.github.com/repos/infracost/infracost/releases/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > infracost.tgz && tar -xzf infracost.tgz && rm infracost.tgz && sudo mv infracost-linux-amd64 /usr/bin/infracost && infracost register curl -L "$(curl -s https://api.github.com/repos/minamijoyo/tfupdate/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > tfupdate.tar.gz && tar -xzf tfupdate.tar.gz tfupdate && rm tfupdate.tar.gz && sudo mv tfupdate /usr/bin/ +curl -L "$(curl -s https://api.github.com/repos/minamijoyo/hcledit/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > hcledit.tar.gz && tar -xzf hcledit.tar.gz hcledit && rm hcledit.tar.gz && sudo mv hcledit /usr/bin/ ```
@@ -157,6 +161,7 @@ curl -L "$(curl -s https://api.github.com/repos/aquasecurity/tfsec/releases/late sudo apt install -y jq && \ curl -L "$(curl -s https://api.github.com/repos/infracost/infracost/releases/latest | grep -o -E -m 1 "https://.+?-linux-amd64.tar.gz")" > infracost.tgz && tar -xzf infracost.tgz && rm infracost.tgz && sudo mv infracost-linux-amd64 /usr/bin/infracost && infracost register curl -L "$(curl -s https://api.github.com/repos/minamijoyo/tfupdate/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > tfupdate.tar.gz && tar -xzf tfupdate.tar.gz tfupdate && rm tfupdate.tar.gz && sudo mv tfupdate /usr/bin/ +curl -L "$(curl -s https://api.github.com/repos/minamijoyo/hcledit/releases/latest | grep -o -E -m 1 "https://.+?_linux_amd64.tar.gz")" > hcledit.tar.gz && tar -xzf hcledit.tar.gz hcledit && rm hcledit.tar.gz && sudo mv hcledit /usr/bin/ ``` @@ -217,8 +222,8 @@ There are several [pre-commit](https://pre-commit.com/) hooks to keep Terraform | Hook name | Description | Dependencies
[Install instructions here](#1-install-dependencies) | -| ------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | -| `checkov` and `terraform_checkov` | [checkov](https://github.com/bridgecrewio/checkov) static analysis of terraform templates to spot potential security issues. [Hook notes](#checkov-deprecated-and-terraform_checkov) | `checkov`
Ubuntu deps: `python3`, `python3-pip` | +| ------------------------------------------------------ |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| `checkov` and `terraform_checkov` | [checkov](https://github.com/bridgecrewio/checkov) static analysis of terraform templates to spot potential security issues. [Hook notes](#checkov-deprecated-and-terraform_checkov) | `checkov`
Ubuntu deps: `python3`, `python3-pip` | | `infracost_breakdown` | Check how much your infra costs with [infracost](https://github.com/infracost/infracost). [Hook notes](#infracost_breakdown) | `infracost`, `jq`, [Infracost API key](https://www.infracost.io/docs/#2-get-api-key) | | `terraform_docs` | Inserts input and output documentation into `README.md`. Recommended. [Hook notes](#terraform_docs) | `terraform-docs` | | `terraform_docs_replace` | Runs `terraform-docs` and pipes the output directly to README.md. **DEPRECATED**, see [#248](https://github.com/antonbabenko/pre-commit-terraform/issues/248). [Hook notes](#terraform_docs_replace-deprecated) | `python3`, `terraform-docs` | @@ -230,6 +235,7 @@ There are several [pre-commit](https://pre-commit.com/) hooks to keep Terraform | `terraform_validate` | Validates all Terraform configuration files. [Hook notes](#terraform_validate) | - | | `terragrunt_fmt` | Reformat all [Terragrunt](https://github.com/gruntwork-io/terragrunt) configuration files (`*.hcl`) to a canonical format. | `terragrunt` | | `terragrunt_validate` | Validates all [Terragrunt](https://github.com/gruntwork-io/terragrunt) configuration files (`*.hcl`) | `terragrunt` | +| `terraform_wrapper_module_for_each` | Generates Terraform wrappers with `for_each` in module. [Hook notes](#terraform_wrapper_module_for_each) | `hcledit` | | `terrascan` | [terrascan](https://github.com/accurics/terrascan) Detect compliance and security violations. [Hook notes](#terrascan) | `terrascan` | | `tfupdate` | [tfupdate](https://github.com/minamijoyo/tfupdate) Update version constraints of Terraform core, providers, and modules. [Hook notes](#tfupdate) | `tfupdate` | @@ -632,6 +638,27 @@ Example: **Note:** The latter method will leave an "aliased-providers.tf.json" file in your repo. You will either want to automate a way to clean this up or add it to your `.gitignore` or both. +### terraform_wrapper_module_for_each + +`terraform_wrapper_module_for_each` generates module wrappers for Terraform modules (useful for Terragrunt where `for_each` is not supported). When using this hook without arguments it will create wrappers for the root module and all modules available in "modules" directory. + +You may want to customize some of the options: + +1. `--module-dir=...` - Specify a single directory to process. Values: "." (means just root module), "modules/iam-user" (a single module), or empty (means include all submodules found in "modules/*"). +2. `--module-repo-org=...` - Module repository organization. +3. `--module-repo-shortname=...` - Short name of the repository (e.g. "s3-bucket"). +4. `--module-repo-provider=...` - Name of the repository provider (e.g. "aws" or "google"). + +Sample configuration: + +```yaml +- id: terraform_wrapper_module_for_each + args: + - --args=--module-dir=. # Process only root module + - --args=--dry-run # No files will be created/updated + - --args=--verbose # Verbose output +``` + ### terrascan 1. `terrascan` supports custom arguments so you can pass supported flags like `--non-recursive` and `--policy-type` to disable recursive inspection and set the policy type respectively: @@ -690,9 +717,7 @@ This repository is managed by [Anton Babenko](https://github.com/antonbabenko) w MIT licensed. See [LICENSE](LICENSE) for full details. -### Additional terms of use for users from Russia and Belarus - -By using the code provided in this repository you agree with the following: +### Additional information for users from Russia and Belarus * Russia has [illegally annexed Crimea in 2014](https://en.wikipedia.org/wiki/Annexation_of_Crimea_by_the_Russian_Federation) and [brought the war in Donbas](https://en.wikipedia.org/wiki/War_in_Donbas) followed by [full-scale invasion of Ukraine in 2022](https://en.wikipedia.org/wiki/2022_Russian_invasion_of_Ukraine). * Russia has brought sorrow and devastations to millions of Ukrainians, killed hundreds of innocent people, damaged thousands of buildings, and forced several million people to flee. diff --git a/hooks/terraform_wrapper_module_for_each.sh b/hooks/terraform_wrapper_module_for_each.sh new file mode 100755 index 000000000..3dd3698d9 --- /dev/null +++ b/hooks/terraform_wrapper_module_for_each.sh @@ -0,0 +1,409 @@ +#!/usr/bin/env bash +set -eo pipefail + +# globals variables +# hook ID, see `- id` for details in .pre-commit-hooks.yaml file +# shellcheck disable=SC2034 # Unused var. +readonly HOOK_ID='terraform_wrapper_for_each_module' +# shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +# shellcheck source=_common.sh +. "$SCRIPT_DIR/_common.sh" + +function main { + common::initialize "$SCRIPT_DIR" + common::parse_cmdline "$@" + # shellcheck disable=SC2153 # False positive + terraform_module_wrapper_ "${ARGS[*]}" +} + +readonly CONTENT_MAIN_TF='module "wrapper" {}' +readonly CONTENT_VARIABLES_TF='variable "defaults" { + description = "Map of default values which will be used for each item." + type = any + default = {} +} + +variable "items" { + description = "Maps of items to create a wrapper from. Values are passed through to the module." + type = any + default = {} +}' +readonly CONTENT_OUTPUTS_TF='output "wrapper" { + description = "Map of outputs of a wrapper." + value = module.wrapper + WRAPPER_OUTPUT_SENSITIVE +}' +readonly CONTENT_VERSIONS_TF='terraform { + required_version = ">= 0.13.1" +}' +# shellcheck disable=SC2016 # False positive +readonly CONTENT_README='# WRAPPER_TITLE + +The configuration in this directory contains an implementation of a single module wrapper pattern, which allows managing several copies of a module in places where using the native Terraform 0.13+ `for_each` feature is not feasible (e.g., with Terragrunt). + +You may want to use a single Terragrunt configuration file to manage multiple resources without duplicating `terragrunt.hcl` files for each copy of the same module. + +This wrapper does not implement any extra functionality. + +## Usage with Terragrunt + +`terragrunt.hcl`: + +```hcl +terraform { + source = "tfr:///MODULE_REPO_ORG/MODULE_REPO_SHORTNAME/MODULE_REPO_PROVIDER//WRAPPER_PATH" + # Alternative source: + # source = "git::git@github.com:MODULE_REPO_ORG/terraform-MODULE_REPO_PROVIDER-MODULE_REPO_SHORTNAME.git?ref=master//WRAPPER_PATH" +} + +inputs = { + defaults = { # Default values + create = true + tags = { + Terraform = "true" + Environment = "dev" + } + } + + items = { + my-item = { + # omitted... can be any argument supported by the module + } + my-second-item = { + # omitted... can be any argument supported by the module + } + # omitted... + } +} +``` + +## Usage with Terraform + +```hcl +module "wrapper" { + source = "MODULE_REPO_ORG/MODULE_REPO_SHORTNAME/MODULE_REPO_PROVIDER//WRAPPER_PATH" + + defaults = { # Default values + create = true + tags = { + Terraform = "true" + Environment = "dev" + } + } + + items = { + my-item = { + # omitted... can be any argument supported by the module + } + my-second-item = { + # omitted... can be any argument supported by the module + } + # omitted... + } +} +``` + +## Example: Manage multiple S3 buckets in one Terragrunt layer + +`eu-west-1/s3-buckets/terragrunt.hcl`: + +```hcl +terraform { + source = "tfr:///terraform-aws-modules/s3-bucket/aws//wrappers" + # Alternative source: + # source = "git::git@github.com:terraform-aws-modules/terraform-aws-s3-bucket.git?ref=master//wrappers" +} + +inputs = { + defaults = { + force_destroy = true + + attach_elb_log_delivery_policy = true + attach_lb_log_delivery_policy = true + attach_deny_insecure_transport_policy = true + attach_require_latest_tls_policy = true + } + + items = { + bucket1 = { + bucket = "my-random-bucket-1" + } + bucket2 = { + bucket = "my-random-bucket-2" + tags = { + Secure = "probably" + } + } + } +} +```' + +terraform_module_wrapper_() { + local args + read -r -a args <<< "$1" + + check_dependencies + + local root_dir + local module_dir="" # values: empty (default), "." (just root module), or a single module (e.g. "modules/iam-user") + local wrapper_dir="wrappers" + local wrapper_relative_source_path="../" # From "wrappers" to root_dir. @todo: Count relative path for wrapper_dir with multiple directories + local module_repo_org="terraform-aws-modules" # @todo: set all module_repo_* values from env vars, if specified + local module_repo_name + local module_repo_shortname + local module_repo_provider="aws" + local dry_run="false" + local verbose="false" + + root_dir=$(git rev-parse --show-toplevel 2> /dev/null || pwd) + module_repo_name=$(basename "$root_dir") + module_repo_shortname="${module_repo_name//terraform-aws-/}" + + for argv in "${args[@]}"; do + + IFS="=" read -r -a onearg <<< "$argv" + + local key="${onearg[0]}" + local value="${onearg[1]}" + + case "$key" in + --root-dir) + root_dir="$value" + ;; + --module-dir) + module_dir="$value" + ;; + --wrapper-dir) + wrapper_dir="$value" + ;; + --module-repo-org) + module_repo_org="$value" + ;; + --module-repo-shortname) + module_repo_shortname="$value" + ;; + --module-repo-provider) + module_repo_provider="$value" + ;; + --dry-run) + dry_run="true" + ;; + --verbose) + verbose="true" + ;; + *) + echo "ERROR: Unrecognized argument: $key" + echo "Hook ID: $HOOK_ID." + echo "Generate Terraform module wrapper. Available arguments:" + echo "--root-dir=... - Root dir of the repository (Optional)" + echo "--module-dir=... - Single module directory. Values: \".\" (means just root module), \"modules/iam-user\" (a single module), or empty (means include all submodules found in \"modules/*\"). Default: \"${module_dir}\". (Optional)" + echo "--wrapper-dir=... - Directory where 'wrappers' should be saved. Default: \"${wrapper_dir}\". (Optional)" + echo "--module-repo-org=... - Module repository organization (e.g., 'terraform-aws-modules'). (Optional)" + echo "--module-repo-shortname=... - Short name of the repository (e.g., for 'terraform-aws-s3-bucket' it should be 's3-bucket'). (Optional)" + echo "--module-repo-provider=... - Name of the repository provider (e.g., for 'terraform-aws-s3-bucket' it should be 'aws'). (Optional)" + echo "--dry-run - Whether to run in dry mode. By default, files will be overwritten." + echo "--verbose - Show verbose output." + echo + echo "Example:" + echo "--module-dir=modules/object - Generate wrapper for one specific submodule." + echo "--module-dir=. - Generate wrapper for the root module." + echo "--module-repo-org=terraform-google-modules --module-repo-shortname=network --module-repo-provider=google - Generate wrappers for repository available by name \"terraform-google-modules/network/google\" in the Terraform registry and it includes all modules (root and in \"modules/*\")." + exit 1 + ;; + esac + + done + + if [[ -z "$root_dir" ]]; then + echo "--root-dir can't be empty. Remove it to use default value." + exit 1 + fi + + if [[ -z "$wrapper_dir" ]]; then + echo "--wrapper-dir can't be empty. Remove it to use default value." + exit 1 + fi + + if [[ -z "$module_repo_org" ]]; then + echo "--module-repo-org can't be empty. Remove it to use default value." + exit 1 + fi + + if [[ -z "$module_repo_shortname" ]]; then + echo "--module-repo-shortname can't be empty. It should be part of full repo name (eg, s3-bucket)." + exit 1 + fi + + if [[ -z "$module_repo_provider" ]]; then + echo "--module-repo-provider can't be empty. It should be name of the provider used by the module (eg, aws)." + exit 1 + fi + + if [ ! -d "$root_dir" ]; then + echo "Root directory $root_dir does not exist!" + exit 1 + fi + + declare -a all_module_dirs=("./") + # Find all modules directories if nothing was provided via "--module-dir" argument + if [[ -z "$module_dir" ]]; then + # shellcheck disable=SC2207 + all_module_dirs+=($(cd "${root_dir}" && find . -path '**/modules/*' -maxdepth 2 -type d -print)) + else + all_module_dirs=("$module_dir") + fi + + for module_dir in "${all_module_dirs[@]}"; do + + # Remove "./" from the "./modules/iam-user" or "./" + module_dir="${module_dir//.\//}" + + full_module_dir="${root_dir}/${module_dir}" + # echo "FULL=${full_module_dir}" + + if [ ! -d "$full_module_dir" ]; then + echo "Module directory $full_module_dir does not exist!" + exit 1 + fi + + # Remove "modules/" from "modules/iam-user" + module_name="${module_dir//modules\//}" + if [ "$module_name" == "" ]; then + wrapper_title="Wrapper for the root module" + wrapper_path="${wrapper_dir}" + else + wrapper_title="Wrapper for module: \`${module_dir}\`" + wrapper_path="${wrapper_dir}/${module_name}" + fi + + # Wrappers will be stored in "wrappers/{module_name}" + output_dir="${root_dir}/${wrapper_dir}/${module_name}" + + [ ! -d "$output_dir" ] && mkdir -p "$output_dir" + + # Calculate relative depth for module source by number of slashes + module_depth="${module_dir//[^\/]/}" + + local relative_source_path=$wrapper_relative_source_path + + for _ in $(seq 0 ${#module_depth} | tail -n +2); do + relative_source_path+="../" + done + + create_tmp_file_tf + + if [ "$verbose" == "true" ]; then + echo "Root directory: $root_dir" + echo "Module directory: $module_dir" + echo "Output directory: $output_dir" + echo "Temp file: $tmp_file_tf" + echo + fi + + # Read content of all terraform files + # shellcheck disable=SC2207 + all_tf_content=$(find "${full_module_dir}" -name '*.tf' -maxdepth 1 -type f -exec cat {} +) + + if [[ -z "$all_tf_content" ]]; then + common::colorify "yellow" "Skipping ${full_module_dir} because there are no *.tf files." + continue + fi + + # Get names of module variables in all terraform files + # shellcheck disable=SC2207 + declare -a module_vars=($(echo "$all_tf_content" | hcledit block list | grep variable. | cut -d'.' -f 2)) + + # Get names of module outputs in all terraform files + # shellcheck disable=SC2207 + declare -a module_outputs=($(echo "$all_tf_content" | hcledit block list | grep output. | cut -d'.' -f 2)) + + # Looking for sensitive output + local wrapper_output_sensitive="# sensitive = false # No sensitive module output found" + for module_output in "${module_outputs[@]}"; do + module_output_sensitive=$(echo "$all_tf_content" | hcledit attribute get "output.${module_output}.sensitive") + + # At least one output is sensitive - the wrapper's output should be sensitive, too + if [ "$module_output_sensitive" == "true" ]; then + wrapper_output_sensitive="sensitive = true # At least one sensitive module output (${module_output}) found (requires Terraform 0.14+)" + break + fi + done + + # Create content of temporary main.tf file + hcledit attribute append module.wrapper.source "\"${relative_source_path}${module_dir}\"" --newline -f "$tmp_file_tf" -u + hcledit attribute append module.wrapper.for_each var.items --newline -f "$tmp_file_tf" -u + + # Add newline before the first variable in a loop + local newline="--newline" + + for module_var in "${module_vars[@]}"; do + # Get default value for the variable + var_default=$(echo "$all_tf_content" | hcledit attribute get "variable.${module_var}.default") + + # Empty default means that the variable is required + if [ "$var_default" == "" ]; then + var_value="try(each.value.${module_var}, var.defaults.${module_var})" + elif [ "$var_default" == "{" ]; then + # BUG in hcledit ( https://github.com/minamijoyo/hcledit/issues/31 ) which breaks on inline comments + # https://github.com/terraform-aws-modules/terraform-aws-security-group/blob/0bd31aa88339194efff470d3b3f58705bd008db0/rules.tf#L8 + # As a result, wrappers in terraform-aws-security-group module are missing values of the rules variable and is not useful. :( + var_value="try(each.value.${module_var}, var.defaults.${module_var}, {})" + else + var_value="try(each.value.${module_var}, var.defaults.${module_var}, $var_default)" + fi + + hcledit attribute append "module.wrapper.${module_var}" "${var_value}" $newline -f "$tmp_file_tf" -u + + newline="" + done + + if [ "$verbose" == "true" ]; then + cat "$tmp_file_tf" + fi + + if [ "$dry_run" == "false" ]; then + common::colorify "green" "Saving files into ${output_dir}" + + mv "$tmp_file_tf" "${output_dir}/main.tf" + + echo "$CONTENT_VARIABLES_TF" > "${output_dir}/variables.tf" + echo "$CONTENT_VERSIONS_TF" > "${output_dir}/versions.tf" + + echo "$CONTENT_OUTPUTS_TF" > "${output_dir}/outputs.tf" + sed -i.bak "s|WRAPPER_OUTPUT_SENSITIVE|${wrapper_output_sensitive}|g" "${output_dir}/outputs.tf" + rm -rf "${output_dir}/outputs.tf.bak" + + echo "$CONTENT_README" > "${output_dir}/README.md" + sed -i.bak "s#WRAPPER_TITLE#${wrapper_title}#g" "${output_dir}/README.md" + sed -i.bak "s#WRAPPER_PATH#${wrapper_path}#g" "${output_dir}/README.md" + sed -i.bak "s#MODULE_REPO_ORG#${module_repo_org}#g" "${output_dir}/README.md" + sed -i.bak "s#MODULE_REPO_SHORTNAME#${module_repo_shortname}#g" "${output_dir}/README.md" + sed -i.bak "s#MODULE_REPO_PROVIDER#${module_repo_provider}#g" "${output_dir}/README.md" + rm -rf "${output_dir}/README.md.bak" + else + common::colorify "yellow" "There is nothing to save. Remove --dry-run flag to write files." + fi + + done + +} + +check_dependencies() { + if [[ ! $(command -v hcledit) ]]; then + echo "ERROR: The binary 'hcledit' is required by this hook but is not installed or in the system's PATH." + echo "Check documentation: https://github.com/minamijoyo/hcledit" + exit 1 + fi +} + +create_tmp_file_tf() { + # Can't append extension for mktemp, so renaming instead + tmp_file=$(mktemp "${TMPDIR:-/tmp}/tfwrapper-XXXXXXXXXX") + mv "$tmp_file" "$tmp_file.tf" + tmp_file_tf="$tmp_file.tf" + + echo "$CONTENT_MAIN_TF" > "$tmp_file_tf" +} + +[ "${BASH_SOURCE[0]}" != "$0" ] || main "$@" From 0b46b9c9354d249f9121cb9040c65c63198dcff9 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sun, 1 May 2022 21:15:27 +0200 Subject: [PATCH 2/5] Fixed all comments --- hooks/terraform_wrapper_module_for_each.sh | 141 ++++++++++++--------- 1 file changed, 78 insertions(+), 63 deletions(-) diff --git a/hooks/terraform_wrapper_module_for_each.sh b/hooks/terraform_wrapper_module_for_each.sh index 3dd3698d9..b77644881 100755 --- a/hooks/terraform_wrapper_module_for_each.sh +++ b/hooks/terraform_wrapper_module_for_each.sh @@ -4,7 +4,8 @@ set -eo pipefail # globals variables # hook ID, see `- id` for details in .pre-commit-hooks.yaml file # shellcheck disable=SC2034 # Unused var. -readonly HOOK_ID='terraform_wrapper_for_each_module' +HOOK_ID=${0##*/} +readonly HOOK_ID=${HOOK_ID%%.*} # shellcheck disable=SC2155 # No way to assign to readonly variable in separate lines readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" # shellcheck source=_common.sh @@ -13,6 +14,10 @@ readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" function main { common::initialize "$SCRIPT_DIR" common::parse_cmdline "$@" + common::parse_and_export_env_vars + + check_dependencies + # shellcheck disable=SC2153 # False positive terraform_module_wrapper_ "${ARGS[*]}" } @@ -139,26 +144,26 @@ inputs = { } ```' -terraform_module_wrapper_() { +function terraform_module_wrapper_() { local args read -r -a args <<< "$1" - check_dependencies - local root_dir local module_dir="" # values: empty (default), "." (just root module), or a single module (e.g. "modules/iam-user") local wrapper_dir="wrappers" - local wrapper_relative_source_path="../" # From "wrappers" to root_dir. @todo: Count relative path for wrapper_dir with multiple directories - local module_repo_org="terraform-aws-modules" # @todo: set all module_repo_* values from env vars, if specified + local wrapper_relative_source_path="../" # From "wrappers" to root_dir. + local module_repo_org local module_repo_name local module_repo_shortname - local module_repo_provider="aws" + local module_repo_provider local dry_run="false" local verbose="false" root_dir=$(git rev-parse --show-toplevel 2> /dev/null || pwd) - module_repo_name=$(basename "$root_dir") - module_repo_shortname="${module_repo_name//terraform-aws-/}" + module_repo_org="terraform-aws-modules" + module_repo_name=${root_dir##*/} + module_repo_shortname="${module_repo_name#terraform-aws-}" + module_repo_provider="aws" for argv in "${args[@]}"; do @@ -193,83 +198,93 @@ terraform_module_wrapper_() { verbose="true" ;; *) - echo "ERROR: Unrecognized argument: $key" - echo "Hook ID: $HOOK_ID." - echo "Generate Terraform module wrapper. Available arguments:" - echo "--root-dir=... - Root dir of the repository (Optional)" - echo "--module-dir=... - Single module directory. Values: \".\" (means just root module), \"modules/iam-user\" (a single module), or empty (means include all submodules found in \"modules/*\"). Default: \"${module_dir}\". (Optional)" - echo "--wrapper-dir=... - Directory where 'wrappers' should be saved. Default: \"${wrapper_dir}\". (Optional)" - echo "--module-repo-org=... - Module repository organization (e.g., 'terraform-aws-modules'). (Optional)" - echo "--module-repo-shortname=... - Short name of the repository (e.g., for 'terraform-aws-s3-bucket' it should be 's3-bucket'). (Optional)" - echo "--module-repo-provider=... - Name of the repository provider (e.g., for 'terraform-aws-s3-bucket' it should be 'aws'). (Optional)" - echo "--dry-run - Whether to run in dry mode. By default, files will be overwritten." - echo "--verbose - Show verbose output." - echo - echo "Example:" - echo "--module-dir=modules/object - Generate wrapper for one specific submodule." - echo "--module-dir=. - Generate wrapper for the root module." - echo "--module-repo-org=terraform-google-modules --module-repo-shortname=network --module-repo-provider=google - Generate wrappers for repository available by name \"terraform-google-modules/network/google\" in the Terraform registry and it includes all modules (root and in \"modules/*\")." + cat << EOF +ERROR: Unrecognized argument: $key +Hook ID: $HOOK_ID. +Generate Terraform module wrapper. Available arguments: +--root-dir=... - Root dir of the repository (Optional) +--module-dir=... - Single module directory. Options: "." (means just root module), + "modules/iam-user" (a single module), or empty (means include all + submodules found in "modules/*"). Default: "${module_dir}". (Optional) +--wrapper-dir=... - Directory where 'wrappers' should be saved. Default: "${wrapper_dir}". (Optional) +--module-repo-org=... - Module repository organization (e.g., 'terraform-aws-modules'). (Optional) +--module-repo-shortname=... - Short name of the repository (e.g., for 'terraform-aws-s3-bucket' it should be 's3-bucket'). (Optional) +--module-repo-provider=... - Name of the repository provider (e.g., for 'terraform-aws-s3-bucket' it should be 'aws'). (Optional) +--dry-run - Whether to run in dry mode. If not specified, wrapper files will be overwritten. +--verbose - Show verbose output. + +Example: +--module-dir=modules/object - Generate wrapper for one specific submodule. +--module-dir=. - Generate wrapper for the root module. +--module-repo-org=terraform-google-modules --module-repo-shortname=network --module-repo-provider=google - Generate wrappers for repository available by name "terraform-google-modules/network/google" in the Terraform registry and it includes all modules (root and in "modules/*"). +EOF exit 1 ;; esac done - if [[ -z "$root_dir" ]]; then + if [[ ! $root_dir ]]; then echo "--root-dir can't be empty. Remove it to use default value." exit 1 fi - if [[ -z "$wrapper_dir" ]]; then + if [[ ! $wrapper_dir ]]; then echo "--wrapper-dir can't be empty. Remove it to use default value." exit 1 fi - if [[ -z "$module_repo_org" ]]; then + if [[ ! $module_repo_org ]]; then echo "--module-repo-org can't be empty. Remove it to use default value." exit 1 fi - if [[ -z "$module_repo_shortname" ]]; then + if [[ ! $module_repo_shortname ]]; then echo "--module-repo-shortname can't be empty. It should be part of full repo name (eg, s3-bucket)." exit 1 fi - if [[ -z "$module_repo_provider" ]]; then + if [[ ! $module_repo_provider ]]; then echo "--module-repo-provider can't be empty. It should be name of the provider used by the module (eg, aws)." exit 1 fi - if [ ! -d "$root_dir" ]; then + if [[ ! -d "$root_dir" ]]; then echo "Root directory $root_dir does not exist!" exit 1 fi - declare -a all_module_dirs=("./") + OLD_IFS="$IFS" + IFS=$'\n' + + all_module_dirs=("./") # Find all modules directories if nothing was provided via "--module-dir" argument - if [[ -z "$module_dir" ]]; then + if [[ ! $module_dir ]]; then # shellcheck disable=SC2207 - all_module_dirs+=($(cd "${root_dir}" && find . -path '**/modules/*' -maxdepth 2 -type d -print)) + all_module_dirs+=($(cd "${root_dir}" && find . -maxdepth 2 -path '**/modules/*' -type d -print)) else all_module_dirs=("$module_dir") fi + IFS="$OLD_IFS" + for module_dir in "${all_module_dirs[@]}"; do # Remove "./" from the "./modules/iam-user" or "./" - module_dir="${module_dir//.\//}" + module_dir="${module_dir/.\//}" full_module_dir="${root_dir}/${module_dir}" # echo "FULL=${full_module_dir}" - if [ ! -d "$full_module_dir" ]; then - echo "Module directory $full_module_dir does not exist!" + if [[ ! -d "$full_module_dir" ]]; then + echo "Module directory \"$full_module_dir\" does not exist!" exit 1 fi # Remove "modules/" from "modules/iam-user" - module_name="${module_dir//modules\//}" - if [ "$module_name" == "" ]; then + # module_name="${module_dir//modules\//}" + module_name="${module_dir#modules/}" + if [[ ! $module_name ]]; then wrapper_title="Wrapper for the root module" wrapper_path="${wrapper_dir}" else @@ -280,20 +295,20 @@ terraform_module_wrapper_() { # Wrappers will be stored in "wrappers/{module_name}" output_dir="${root_dir}/${wrapper_dir}/${module_name}" - [ ! -d "$output_dir" ] && mkdir -p "$output_dir" + [[ ! -d "$output_dir" ]] && mkdir -p "$output_dir" # Calculate relative depth for module source by number of slashes module_depth="${module_dir//[^\/]/}" local relative_source_path=$wrapper_relative_source_path - for _ in $(seq 0 ${#module_depth} | tail -n +2); do + for ((c = 0; c < ${#module_depth}; c++)); do relative_source_path+="../" done create_tmp_file_tf - if [ "$verbose" == "true" ]; then + if [[ "$verbose" == "true" ]]; then echo "Root directory: $root_dir" echo "Module directory: $module_dir" echo "Output directory: $output_dir" @@ -305,18 +320,18 @@ terraform_module_wrapper_() { # shellcheck disable=SC2207 all_tf_content=$(find "${full_module_dir}" -name '*.tf' -maxdepth 1 -type f -exec cat {} +) - if [[ -z "$all_tf_content" ]]; then + if [[ ! $all_tf_content ]]; then common::colorify "yellow" "Skipping ${full_module_dir} because there are no *.tf files." continue fi # Get names of module variables in all terraform files # shellcheck disable=SC2207 - declare -a module_vars=($(echo "$all_tf_content" | hcledit block list | grep variable. | cut -d'.' -f 2)) + module_vars=($(echo "$all_tf_content" | hcledit block list | grep variable. | cut -d'.' -f 2)) # Get names of module outputs in all terraform files # shellcheck disable=SC2207 - declare -a module_outputs=($(echo "$all_tf_content" | hcledit block list | grep output. | cut -d'.' -f 2)) + module_outputs=($(echo "$all_tf_content" | hcledit block list | grep output. | cut -d'.' -f 2)) # Looking for sensitive output local wrapper_output_sensitive="# sensitive = false # No sensitive module output found" @@ -324,7 +339,7 @@ terraform_module_wrapper_() { module_output_sensitive=$(echo "$all_tf_content" | hcledit attribute get "output.${module_output}.sensitive") # At least one output is sensitive - the wrapper's output should be sensitive, too - if [ "$module_output_sensitive" == "true" ]; then + if [[ "$module_output_sensitive" == "true" ]]; then wrapper_output_sensitive="sensitive = true # At least one sensitive module output (${module_output}) found (requires Terraform 0.14+)" break fi @@ -342,9 +357,9 @@ terraform_module_wrapper_() { var_default=$(echo "$all_tf_content" | hcledit attribute get "variable.${module_var}.default") # Empty default means that the variable is required - if [ "$var_default" == "" ]; then + if [[ ! $var_default ]]; then var_value="try(each.value.${module_var}, var.defaults.${module_var})" - elif [ "$var_default" == "{" ]; then + elif [[ "$var_default" == "{" ]]; then # BUG in hcledit ( https://github.com/minamijoyo/hcledit/issues/31 ) which breaks on inline comments # https://github.com/terraform-aws-modules/terraform-aws-security-group/blob/0bd31aa88339194efff470d3b3f58705bd008db0/rules.tf#L8 # As a result, wrappers in terraform-aws-security-group module are missing values of the rules variable and is not useful. :( @@ -358,12 +373,10 @@ terraform_module_wrapper_() { newline="" done - if [ "$verbose" == "true" ]; then - cat "$tmp_file_tf" - fi + [[ "$verbose" == "true" ]] && cat "$tmp_file_tf" - if [ "$dry_run" == "false" ]; then - common::colorify "green" "Saving files into ${output_dir}" + if [[ "$dry_run" == "false" ]]; then + common::colorify "green" "Saving files into \"${output_dir}\"" mv "$tmp_file_tf" "${output_dir}/main.tf" @@ -375,11 +388,13 @@ terraform_module_wrapper_() { rm -rf "${output_dir}/outputs.tf.bak" echo "$CONTENT_README" > "${output_dir}/README.md" - sed -i.bak "s#WRAPPER_TITLE#${wrapper_title}#g" "${output_dir}/README.md" - sed -i.bak "s#WRAPPER_PATH#${wrapper_path}#g" "${output_dir}/README.md" - sed -i.bak "s#MODULE_REPO_ORG#${module_repo_org}#g" "${output_dir}/README.md" - sed -i.bak "s#MODULE_REPO_SHORTNAME#${module_repo_shortname}#g" "${output_dir}/README.md" - sed -i.bak "s#MODULE_REPO_PROVIDER#${module_repo_provider}#g" "${output_dir}/README.md" + sed -i.bak -e " + s#WRAPPER_TITLE#${wrapper_title}#g + s#WRAPPER_PATH#${wrapper_path}#g + s#MODULE_REPO_ORG#${module_repo_org}#g + s#MODULE_REPO_SHORTNAME#${module_repo_shortname}#g + s#MODULE_REPO_PROVIDER#${module_repo_provider}#g + " "${output_dir}/README.md" rm -rf "${output_dir}/README.md.bak" else common::colorify "yellow" "There is nothing to save. Remove --dry-run flag to write files." @@ -389,15 +404,15 @@ terraform_module_wrapper_() { } -check_dependencies() { - if [[ ! $(command -v hcledit) ]]; then - echo "ERROR: The binary 'hcledit' is required by this hook but is not installed or in the system's PATH." +function check_dependencies() { + if ! command -v hcledit > /dev/null; then + echo "ERROR: The binary 'hcledit' is required by this hook but is not installed or is not in the system's PATH." echo "Check documentation: https://github.com/minamijoyo/hcledit" exit 1 fi } -create_tmp_file_tf() { +function create_tmp_file_tf() { # Can't append extension for mktemp, so renaming instead tmp_file=$(mktemp "${TMPDIR:-/tmp}/tfwrapper-XXXXXXXXXX") mv "$tmp_file" "$tmp_file.tf" @@ -406,4 +421,4 @@ create_tmp_file_tf() { echo "$CONTENT_MAIN_TF" > "$tmp_file_tf" } -[ "${BASH_SOURCE[0]}" != "$0" ] || main "$@" +[[ "${BASH_SOURCE[0]}" != "$0" ]] || main "$@" From 8bc89c7818724c4ae8380529f86fae45b35a554e Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sun, 1 May 2022 21:18:12 +0200 Subject: [PATCH 3/5] Fixed README a bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7f871f28a..2ad16c27e 100644 --- a/README.md +++ b/README.md @@ -664,7 +664,7 @@ Example: You may want to customize some of the options: 1. `--module-dir=...` - Specify a single directory to process. Values: "." (means just root module), "modules/iam-user" (a single module), or empty (means include all submodules found in "modules/*"). -2. `--module-repo-org=...` - Module repository organization. +2. `--module-repo-org=...` - Module repository organization (e.g. "terraform-aws-modules"). 3. `--module-repo-shortname=...` - Short name of the repository (e.g. "s3-bucket"). 4. `--module-repo-provider=...` - Name of the repository provider (e.g. "aws" or "google"). From f357369e411179c13c884af06ad226e61f44206c Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Mon, 2 May 2022 19:26:07 +0200 Subject: [PATCH 4/5] Apply suggestions from code review Co-authored-by: George L. Yermulnik --- hooks/terraform_wrapper_module_for_each.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hooks/terraform_wrapper_module_for_each.sh b/hooks/terraform_wrapper_module_for_each.sh index b77644881..520bb12c2 100755 --- a/hooks/terraform_wrapper_module_for_each.sh +++ b/hooks/terraform_wrapper_module_for_each.sh @@ -144,7 +144,7 @@ inputs = { } ```' -function terraform_module_wrapper_() { +function terraform_module_wrapper_ { local args read -r -a args <<< "$1" @@ -169,8 +169,8 @@ function terraform_module_wrapper_() { IFS="=" read -r -a onearg <<< "$argv" - local key="${onearg[0]}" - local value="${onearg[1]}" + local key="${argv%%=*}" + local value="${argv#*=}" case "$key" in --root-dir) @@ -404,7 +404,7 @@ EOF } -function check_dependencies() { +function check_dependencies { if ! command -v hcledit > /dev/null; then echo "ERROR: The binary 'hcledit' is required by this hook but is not installed or is not in the system's PATH." echo "Check documentation: https://github.com/minamijoyo/hcledit" @@ -412,7 +412,7 @@ function check_dependencies() { fi } -function create_tmp_file_tf() { +function create_tmp_file_tf { # Can't append extension for mktemp, so renaming instead tmp_file=$(mktemp "${TMPDIR:-/tmp}/tfwrapper-XXXXXXXXXX") mv "$tmp_file" "$tmp_file.tf" From c3f45009a909f024b7ddceab834aeb72c88d04ec Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Mon, 2 May 2022 19:31:05 +0200 Subject: [PATCH 5/5] Fixed shell complain --- hooks/terraform_wrapper_module_for_each.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/hooks/terraform_wrapper_module_for_each.sh b/hooks/terraform_wrapper_module_for_each.sh index 520bb12c2..1a7abed28 100755 --- a/hooks/terraform_wrapper_module_for_each.sh +++ b/hooks/terraform_wrapper_module_for_each.sh @@ -167,8 +167,6 @@ function terraform_module_wrapper_ { for argv in "${args[@]}"; do - IFS="=" read -r -a onearg <<< "$argv" - local key="${argv%%=*}" local value="${argv#*=}"