From 050fa9e33e45d3de1b35fcccff6da9054fa0dbdc Mon Sep 17 00:00:00 2001 From: Tanna McClure Date: Tue, 31 Jan 2023 17:29:45 -0800 Subject: [PATCH 1/4] add super easy test command script --- test/README.md | 87 ++++++++++++++++++-- test/script/run_tests | 187 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 6 deletions(-) create mode 100755 test/script/run_tests diff --git a/test/README.md b/test/README.md index b60d1982ed..fb7714bf2e 100644 --- a/test/README.md +++ b/test/README.md @@ -6,15 +6,36 @@ We use Minitest for the Ruby agent. The following command runs the unit tests w bundle exec rake test -## Running Specific Tests -You can also run a single unit test file like this: +## Super Easy Testing Setup - bundle exec ruby test/new_relic/agent_test.rb +There is a script in test/script/run_tests that makes it easy to run the unit, env, and multiverse tests. -And to run a single test within that file (note that when using the -n argument, you can either supply the entire test name as a string or a partial name as a regex): +Examples + + ./test/script/run_tests -u # unit tests + ./test/script/run_tests -e 61 # env tests running rails61 + ./test/script/run_tests -m rake # multiverse tests running rake in all envs and prepend/chain + ./test/script/run_tests -q rake # multiverse tests running rake only env 0 and prepend + +If you set up shell alias for these, it will make it super convenient to run + +add these aliases in ~/.bash_profile (or ~/.zsh_profile, whatever you use) + + alias bert="./test/script/run_tests -u" + alias bere="./test/script/run_tests -e" + alias berm="./test/script/run_tests -m" + alias bermq="./test/script/run_tests -q" + alias ber="./test/script/run_tests" + +Then you'll be able to run the tests super easy like + + bert # run all unit tests + bere 61 # run env tests for rails 6.1 + berm rake # run all rake multiverse suites + bermq rake # run multiverse rake env=0 method=prepend + ber -h # explains all the args for each option - bundle exec ruby test/new_relic/agent_test.rb -n /test_shutdown/ ## Running Tests in a Rails Environment @@ -26,7 +47,61 @@ In CI, these unit tests are run against all supported major.minor versions of Ra bundle exec rake test:env -### Running Specific Tests + + +## Running Specific Tests + +These env variables work for both the unit tests and env tests. + +Running a specific test file + + TEST="path/to/test_file_you_want_to_run.rb" bundle exec rake test + + +Running a specific test by name + + TESTOPTS="--name=test_name_of_test_to_run" bundle exec rake test + + +You can also specify both + + TEST="path/to/test_file_you_want_to_run.rb" TESTOPTS="--name=test_name_of_test_to_run" bundle exec rake test + + +## Specify a seed + +If you're running into intermittent failures that seem to be related to the order tests are run in, you can specify a seed to the randomization + + TESTOPTS="--seed=12345" bundle exec rake test + + +## Multiverse + +To run a multiverse suite + + bundle exec rake test:multiverse[suite_name] + +More detailed multiverse information available in the [multiverse readme](./multiverse/README.md) + + + + + +## Other Ways To Do Things + +### Unit Tests Only - Specify File and Test Name +This doesn't work for the env tests, but this is also an option when running just the unit tests. + +You can also run a single unit test file like this: + + bundle exec ruby test/new_relic/agent_test.rb + +And to run a single test within that file (note that when using the -n argument, you can either supply the entire test name as a string or a partial name as a regex): + + bundle exec ruby test/new_relic/agent_test.rb -n /test_shutdown/ + + +### Env Tests Only - Specify File The file environment variable can be added to the test:env invocation to run a specific unit file. It can be exact file name, or a wildcard pattern. Multiple file patterns can be specified by separating with a comma with no spaces surrounding: diff --git a/test/script/run_tests b/test/script/run_tests new file mode 100755 index 0000000000..4059361535 --- /dev/null +++ b/test/script/run_tests @@ -0,0 +1,187 @@ +#!/bin/bash + +help() { + echo "Runs the unit, env, or multiverse tests" + echo + echo "Syntax: run_tests [-u|e|m|q] arg1 arg2 arg3" + echo " options:" + echo " -u Runs unit tests" + echo " optional arg 1: file_path OR test_name" + echo " optional arg 2: test_name (if also passing in file_path in arg 1)" + echo " Examples: " + echo " ./test/script/run_tests -u" + echo " ./test/script/run_tests -u test_name" + echo " ./test/script/run_tests -u path/to/test_file.rb" + echo " ./test/script/run_tests -u path/to/test_file.rb test_name" + echo " " + echo " -e Runs env tests" + echo " arg 1: rails version to test. The leading 'rails' can be left off. 61,70 works like rails61,rails70" + echo " optional arg 2: file_path OR test_name" + echo " optional arg 3: test_name (if also passing in file_path in arg 2)" + echo " Examples: " + echo " ./test/script/run_tests -e 61" + echo " ./test/script/run_tests -e 61,71 test_name" + echo " ./test/script/run_tests -e rails61 path/to/test_file.rb" + echo " ./test/script/run_tests -e rails61,rails71 path/to/test_file.rb test_name" + echo " " + echo " -m Runs multiverse tests" + echo " arg 1: multiverse suite to run (or any other args you wanna pass in to multiverse)" + echo " the 'debug' multiverse arg is automatically included" + echo " optional arg 2: file_path OR test_name" + echo " optional arg 3: test_name (if also passing in file_path in arg 2)" + echo " Examples: " + echo " ./test/script/run_tests -m rake" + echo " ./test/script/run_tests -m rake,env=2" + echo " ./test/script/run_tests -m rake test_name" + echo " ./test/script/run_tests -m rake path/to/test_file.rb" + echo " ./test/script/run_tests -m rake path/to/test_file.rb test_name" + echo " " + echo " -q Runs quick multiverse tests. Prepend method only and env 0, unless env number is passed in" + echo " arg 1: multiverse suite to run (or any other args you wanna pass in to multiverse)" + echo " the 'debug' multiverse arg is automatically included" + echo " optional arg 2: env# OR file_path OR test_name" + echo " optional arg 3: file_path OR test_name (if also passing in env # in arg 2)" + echo " optional arg 4: test_name (if also passing in file_path in arg 3)" + echo " Examples: " + echo " ./test/script/run_tests -m rake" + echo " ./test/script/run_tests -m rake 2" + echo " ./test/script/run_tests -m rake test_name" + echo " ./test/script/run_tests -m rake path/to/test_file.rb" + echo " ./test/script/run_tests -m rake path/to/test_file.rb test_name" + echo " " +} + + +unit_command() { + # echo "ENV TEST="$TEST" TESTOPTS="$TESTOPTS"" + # echo "bundle exec rake test -q;" + bundle exec rake test -q; +} + +env_test_command() { + # echo "ENV TEST="$TEST" TESTOPTS="$TESTOPTS"" + # echo "bundle exec rake test:env["$@"];" + bundle exec rake test:env["$@"]; + } + +multiverse_command() { + # echo "ENV TEST="$TEST" TESTOPTS="$TESTOPTS"" + # echo "bundle exec rake test:multiverse["$@",debug];" + bundle exec rake test:multiverse["$@",debug]; +} + +# doesn't overwrite testopts that already exist, +# that way you can still pass in a seed if you want +set_test_opts() { + test_opts_result=""$TESTOPTS" --name="$1"" +} + +# organizes the args for mutiverse and calls the command +run_multiverse() { + if [[ -n "$3" ]]; then + # echo "running file and name" + multiverse_command "$1",file="$2",name="$3"; + elif [[ -n "$2" && "$2" =~ ^test_ ]]; then + # echo "running name" + multiverse_command "$1",name="$2"; + elif [[ -n "$2" ]]; then + # echo "running file" + multiverse_command "$1",file="$2"; + else + # echo "running suite $1" + multiverse_command "$1"; + fi +} + +# calls multiverse but only env 0 or specified and method prepend +run_multiverse_quick() { + if [[ "$2" =~ [0-9]+ && ! "$2" =~ ^test ]]; then + run_multiverse "$1",env="$2",method=prepend "$3" "$4" + else + run_multiverse "$1",env=0,method=prepend "$2" "$3" + fi +} + +# organizes the args for env tests and calls the command +run_env_tests() { + # this will add "rails" before any number so you can pass in just "61" or "61,70" + ENVARGS=$(echo "$1" | sed -E -e 's/(\,)|(\,rails)/,rails/g' | sed '/^rails/!s/^/rails/'); + + if [[ -n "$3" ]]; then + # echo "running file and name" + set_test_opts $3 + TEST="../../../$2" TESTOPTS="$test_opts_result" env_test_command "$ENVARGS"; + elif [[ -n "$2" && "$2" =~ ^test_ ]]; then + # echo "running name" + set_test_opts $2 + TESTOPTS="$test_opts_result" env_test_command "$ENVARGS"; + elif [[ -n "$2" ]]; then + # echo "running file" + TEST="../../../$2" env_test_command "$ENVARGS"; + else + # echo "running env tests" + env_test_command "$ENVARGS"; + fi +} + +# organizes the args for the unit tests and calls the command +run_unit_tests() { + if [[ -n "$2" ]]; then + # echo "running file and name" + set_test_opts $2 + TEST="$1" TESTOPTS="$test_opts_result" unit_command; + elif [[ -n "$1" && "$1" =~ ^test_ ]]; then + # echo "running name" + set_test_opts $2 + TESTOPTS="$test_opts_result" unit_command; + elif [[ -n "$1" ]]; then + # echo "running file" + TEST="$1" unit_command; + else + # echo "running unit tests" + unit_command; + fi +} + +# helper method to output the help info and error and then exit the script +output_arg_error(){ + help + echo $1 + exit +} + +while getopts "huemq" option; do + shift 1 # removes the option (ex: -u) from the args so we can pass all the args with $@ + case $option in + h) + help + exit;; + + u) + run_unit_tests $@ + exit;; + + e) + if ! [[ -n "$1" ]]; then + output_arg_error "ERROR: Must specify at least one rails version" + fi + run_env_tests $@ + exit;; + + m) + if ! [[ -n "$1" ]]; then + output_arg_error "ERROR: Must specify multiverse suite" + fi + run_multiverse $@ + exit;; + + q) + if ! [[ -n "$1" ]]; then + output_arg_error "ERROR: Must specify multiverse suite" + fi + run_multiverse_quick $@ + exit;; + esac +done + +help \ No newline at end of file From b75f04c5e0987a0c679841588c80ab8f5571652b Mon Sep 17 00:00:00 2001 From: Tanna McClure Date: Wed, 1 Feb 2023 08:48:52 -0800 Subject: [PATCH 2/4] fix zshenv in readme --- test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index fb7714bf2e..b26bf62210 100644 --- a/test/README.md +++ b/test/README.md @@ -20,7 +20,7 @@ Examples If you set up shell alias for these, it will make it super convenient to run -add these aliases in ~/.bash_profile (or ~/.zsh_profile, whatever you use) +add these aliases in ~/.bash_profile (or ~/.zshenv, depending on what shell you use) alias bert="./test/script/run_tests -u" alias bere="./test/script/run_tests -e" From 28ebca79526867e8d81c928264aabf111638f8c3 Mon Sep 17 00:00:00 2001 From: Tanna McClure Date: Wed, 1 Feb 2023 08:59:48 -0800 Subject: [PATCH 3/4] add info to readme about using full path for alias --- test/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/README.md b/test/README.md index b26bf62210..bd81fd22ff 100644 --- a/test/README.md +++ b/test/README.md @@ -28,6 +28,8 @@ add these aliases in ~/.bash_profile (or ~/.zshenv, depending on what shell you alias bermq="./test/script/run_tests -q" alias ber="./test/script/run_tests" +Also, if you set it to the full path instead of relative, then you can run it from anywhere and it will work in wherever you are (for example, if you've `cd`-ed into infinite_tracing and run `bert` it will run the infinite tracing tests if you're using the full path, but error if its relative) + Then you'll be able to run the tests super easy like bert # run all unit tests From 968a543e767e4182d229ba3de6a12b9d312be6c9 Mon Sep 17 00:00:00 2001 From: Tanna McClure Date: Wed, 1 Feb 2023 09:45:19 -0800 Subject: [PATCH 4/4] fix unit test test name arg --- test/script/run_tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/script/run_tests b/test/script/run_tests index 4059361535..b5c622387b 100755 --- a/test/script/run_tests +++ b/test/script/run_tests @@ -132,7 +132,7 @@ run_unit_tests() { TEST="$1" TESTOPTS="$test_opts_result" unit_command; elif [[ -n "$1" && "$1" =~ ^test_ ]]; then # echo "running name" - set_test_opts $2 + set_test_opts $1 TESTOPTS="$test_opts_result" unit_command; elif [[ -n "$1" ]]; then # echo "running file"