diff --git a/.editorconfig b/.editorconfig index 74f9834a3..b692705e2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,7 +7,7 @@ charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true -[{*.{sh,py,md},Dockerfile}] +[{*.{py,md},Dockerfile}] indent_size = 4 [*.md] diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 000000000..10ba27397 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,81 @@ +# Notes for contributors + +1. Python hooks are supported now too. All you have to do is: + 1. add a line to the `console_scripts` array in `entry_points` in `setup.py` + 2. Put your python script in the `pre_commit_hooks` folder + +Enjoy the clean, valid, and documented code! + +* [Run and debug hooks locally](#run-and-debug-hooks-locally) +* [Run hook performance test](#run-hook-performance-test) + * [Run via BASH](#run-via-bash) + * [Run via Docker](#run-via-docker) + * [Check results](#check-results) + * [Cleanup](#cleanup) + +## Run and debug hooks locally + +```bash +pre-commit try-repo {-a} /path/to/local/pre-commit-terraform/repo {hook_name} +``` + +I.e. + +```bash +pre-commit try-repo /mnt/c/Users/tf/pre-commit-terraform terraform_fmt # Run only `terraform_fmt` check +pre-commit try-repo -a ~/pre-commit-terraform # run all existing checks from repo +``` + +Running `pre-commit` with `try-repo` ignores all arguments specified in `.pre-commit-config.yaml`. + +## Run hook performance test + +To check is your improvement not violate performance, we have dummy execution time tests. + +Script accept next options: + +| # | Name | Example value | Description | +| --- | ---------------------------------- | ------------------------------------------------------------------------ | ---------------------------------------------------- | +| 1 | `TEST_NUM` | `200` | How many times need repeat test | +| 2 | `TEST_COMMAND` | `'pre-commit try-repo -a /tmp/159/pre-commit-terraform terraform_tfsec'` | Valid pre-commit command | +| 3 | `TEST_DIR` | `'/tmp/infrastructure'` | Dir on what you run tests. | +| 4 | `TEST_DESCRIPTION` | ```'`terraform_tfsec` PR #123:'``` | Text that you'd like to see in result | +| 5 | `RAW_TEST_`
`RESULTS_FILE_NAME` | `terraform_tfsec_pr123` | (Temporary) File where all test data will be stored. | + +### Run via BASH + +```bash +# Install deps +sudo apt install -y datamash +# Run tests +./hooks_performance_test.sh 200 'pre-commit try-repo -a /tmp/159/pre-commit-terraform terraform_tfsec' '/tmp/infrastructure' '`terraform_tfsec` v1.51.0:' 'terraform_tfsec_pr159' +``` + +### Run via Docker + +```bash +# Build `pre-commit` image +docker build -t pre-commit --build-arg INSTALL_ALL=true . +# Build test image +docker build -t pre-commit-tests tests/ +# Run +TEST_NUM=1 +TEST_DIR='/tmp/infrastructure' +PRE_COMMIT_DIR="$(pwd)" +TEST_COMMAND='pre-commit try-repo -a /pct terraform_tfsec' +TEST_DESCRIPTION='`terraform_tfsec` v1.51.0:' +RAW_TEST_RESULTS_FILE_NAME='terraform_tfsec_pr159' + +docker run -v "$PRE_COMMIT_DIR:/pct:rw" -v "$TEST_DIR:/lint:ro" pre-commit-tests \ + $TEST_NUM "$TEST_COMMAND" '/lint' "$RAW_TEST_RESULTS_FILE_NAME" "$RAW_TEST_RESULTS_FILE_NAME" +``` + +### Check results + +Results will be located at `./test/results` dir. + +### Cleanup + +```bash +sudo rm -rf tests/results +``` diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..0bbeada90 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tests/results/* diff --git a/README.md b/README.md index 37e05493c..116cfdf7a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![Github tag](https://img.shields.io/github/tag/antonbabenko/pre-commit-terraform.svg)](https://github.com/antonbabenko/pre-commit-terraform/releases) ![maintenance status](https://img.shields.io/maintenance/yes/2021.svg) [![Help Contribute to Open Source](https://www.codetriage.com/antonbabenko/pre-commit-terraform/badges/users.svg)](https://www.codetriage.com/antonbabenko/pre-commit-terraform) +Want to Contribute? Check [open issues](https://github.com/antonbabenko/pre-commit-terraform/issues?q=label%3A%22good+first+issue%22+is%3Aopen+sort%3Aupdated-desc) and [contributing notes](/.github/CONTRIBUTING.md). + * [How to install](#how-to-install) * [1. Install dependencies](#1-install-dependencies) * [2. Install the pre-commit hook globally](#2-install-the-pre-commit-hook-globally) @@ -15,8 +17,6 @@ * [terraform_tflint](#terraform_tflint) * [terraform_tfsec](#terraform_tfsec) * [terraform_validate](#terraform_validate) -* [Notes for contributors](#notes-for-contributors) - * [Run and debug hooks locally](#run-and-debug-hooks-locally) * [Authors](#authors) * [License](#license) @@ -330,28 +330,6 @@ Example: **Warning:** If you use Terraform workspaces, DO NOT use this workaround ([details](https://github.com/antonbabenko/pre-commit-terraform/issues/203#issuecomment-918791847)). Wait to [`force-init`](https://github.com/antonbabenko/pre-commit-terraform/issues/224) option implementation -## Notes for contributors - -1. Python hooks are supported now too. All you have to do is: - 1. add a line to the `console_scripts` array in `entry_points` in `setup.py` - 2. Put your python script in the `pre_commit_hooks` folder - -Enjoy the clean, valid, and documented code! - -### Run and debug hooks locally - -```bash -pre-commit try-repo {-a} /path/to/local/pre-commit-terraform/repo {hook_name} -``` - -I.e. - -```bash -pre-commit try-repo /mnt/c/Users/tf/pre-commit-terraform terraform_fmt # Run only `terraform_fmt` check -pre-commit try-repo -a ~/pre-commit-terraform # run all existing checks from repo -``` - -Running `pre-commit` with `try-repo` ignores all arguments specified in `.pre-commit-config.yaml`. ## Authors diff --git a/tests/Dockerfile b/tests/Dockerfile new file mode 100644 index 000000000..cc0f7d2e1 --- /dev/null +++ b/tests/Dockerfile @@ -0,0 +1,11 @@ +FROM pre-commit + +RUN apt update && \ + apt install -y \ + datamash \ + time && \ + # Cleanup + rm -rf /var/lib/apt/lists/* + +WORKDIR /pct +ENTRYPOINT [ "/pct/tests/hooks_performance_test.sh" ] diff --git a/tests/hooks_performance_test.sh b/tests/hooks_performance_test.sh new file mode 100755 index 000000000..77a353f4a --- /dev/null +++ b/tests/hooks_performance_test.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +TEST_NUM=$1 # 1000 +TEST_COMMAND=$2 # 'pre-commit try-repo -a /tmp/159/pre-commit-terraform terraform_tfsec' +TEST_DIR=$3 # '/tmp/infrastructure' +TEST_DESCRIPTION="$TEST_NUM runs '$4'" # '`terraform_tfsec` PR #123:' +RAW_TEST_RESULTS_FILE_NAME=$5 # terraform_tfsec_pr123 + +function run_tests { + local TEST_NUM=$1 + local TEST_DIR=$2 + local TEST_COMMAND + IFS=" " read -r -a TEST_COMMAND <<< "$3" + local FILE_NAME_TO_SAVE_TEST_RESULTS=$4 + + local RESULTS_DIR + RESULTS_DIR="$(pwd)/tests/results" + + cd "$TEST_DIR" || { echo "Specified TEST_DIR does not exist" && exit 1; } + + for ((i = 1; i <= TEST_NUM; i++)); do + { + echo -e "\n\nTest run $i times\n\n" + /usr/bin/time --quiet -f '%U user %S system %P cpu %e total' \ + "${TEST_COMMAND[@]}" + } 2>> "$RESULTS_DIR/$FILE_NAME_TO_SAVE_TEST_RESULTS" + done + # shellcheck disable=2164 # Always exist + cd - > /dev/null +} + +function generate_table { + local FILE_PATH="tests/results/$1" + + local users_seconds system_seconds cpu total_time + users_seconds=$(awk '{ print $1; }' "$FILE_PATH") + system_seconds=$(awk '{ print $3; }' "$FILE_PATH") + cpu=$(awk '{ gsub("%","",$5); print $5; }' "$FILE_PATH") + total_time=$(awk '{ print $7; }' "$FILE_PATH") + + echo " +| time command | max | min | mean | median | +| -------------- | ------ | ------ | -------- | ------ | +| users seconds | $( + printf %"s\n" $users_seconds | datamash max 1 + ) | $( + printf %"s\n" $users_seconds | datamash min 1 + ) | $( + printf %"s\n" $users_seconds | datamash mean 1 + ) | $(printf %"s\n" $users_seconds | datamash median 1) | +| system seconds | $( + printf %"s\n" $system_seconds | datamash max 1 + ) | $( + printf %"s\n" $system_seconds | datamash min 1 + ) | $( + printf %"s\n" $system_seconds | datamash mean 1 + ) | $(printf %"s\n" $system_seconds | datamash median 1) | +| CPU % | $( + printf %"s\n" $cpu | datamash max 1 + ) | $( + printf %"s\n" $cpu | datamash min 1 + ) | $( + printf %"s\n" $cpu | datamash mean 1 + ) | $(printf %"s\n" $cpu | datamash median 1) | +| Total time | $( + printf %"s\n" $total_time | datamash max 1 + ) | $( + printf %"s\n" $total_time | datamash min 1 + ) | $( + printf %"s\n" $total_time | datamash mean 1 + ) | $(printf %"s\n" $total_time | datamash median 1) | +" +} + +function save_result { + local DESCRIPTION=$1 + local TABLE=$2 + local TEST_RUN_START_TIME=$3 + local TEST_RUN_END_TIME=$4 + + local FILE_NAME=${5:-"tests_result.md"} + + echo -e "\n$DESCRIPTION" >> "tests/results/$FILE_NAME" + echo -e "$TABLE" >> "tests/results/$FILE_NAME" + # shellcheck disable=SC2016,SC2128 # Irrelevant + echo -e ' +
Run details + +* Test Start: '"$TEST_RUN_START_TIME"' +* Test End: '"$TEST_RUN_END_TIME"' + +| Variable name | Value | +| ---------------------------- | --- | +| `TEST_NUM` | '"$TEST_NUM"' | +| `TEST_COMMAND` | '"$TEST_COMMAND"' | +| `TEST_DIR` | '"$TEST_DIR"' | +| `TEST_DESCRIPTION` | '"$TEST_DESCRIPTION"' | +| `RAW_TEST_RESULTS_FILE_NAME` | '"$RAW_TEST_RESULTS_FILE_NAME"' | + +Memory info (`head -n 6 /proc/meminfo`): + +```bash +'"$(head -n 6 /proc/meminfo)"' +``` + +CPU info: + +```bash +Real procs: '"$(grep ^cpu\\scores /proc/cpuinfo | uniq | awk '{print $4}')"' +Virtual (hyper-threading) procs: '"$(grep -c ^processor /proc/cpuinfo)"' +'"$(tail -n 28 /proc/cpuinfo)"' +``` + +
+' >> "tests/results/$FILE_NAME" + +} + +mkdir -p tests/results +TEST_RUN_START_TIME=$(date -u) +# shellcheck disable=SC2128 # Irrelevant +run_tests "$TEST_NUM" "$TEST_DIR" "$TEST_COMMAND" "$RAW_TEST_RESULTS_FILE_NAME" +TEST_RUN_END_TIME=$(date -u) + +TABLE=$(generate_table "$RAW_TEST_RESULTS_FILE_NAME") +save_result "$TEST_DESCRIPTION" "$TABLE" "$TEST_RUN_START_TIME" "$TEST_RUN_END_TIME"