diff --git a/.github/workflows/executableMonitorTests.yml b/.github/workflows/executableMonitorTests.yml new file mode 100644 index 00000000..e03c6955 --- /dev/null +++ b/.github/workflows/executableMonitorTests.yml @@ -0,0 +1,392 @@ +name: Executable Monitor Tests + +on: + push: + branches: ["**"] + pull_request: + branches: [main, v2] + workflow_dispatch: + +env: + # The bash escape character is \033 + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + +jobs: + + test-exe-monitor-success-cases: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + + - env: + stepName: Install Windows Build tools + name: ${{ env.stepName }} + if: runner.os == 'Windows' + id: install-windows-build-tools + uses: microsoft/setup-msbuild@v1.1 + + - env: + stepName: Install Ubuntu Build Tools + name: ${{ env.stepName }} + if: runner.os == 'Linux' + id: install-ubuntu-build--tools + run: | + # ${{ env.stepName }} + echo -e "::group::${{ env.stepName }}" + sudo apt install build-essential + exitStatus=$? + echo -e "::endgroup::" + if [ $exitStatus -eq 0 ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" + exit 1 + fi + + - env: + stepName: Compile Executable Monitor Test Files + name: ${{ env.stepName }} + id: compile-executable-monitor-test-files + shell: bash + run: | + # ${{ env.stepName }} + echo -e "::group::${{ env.stepName }}" + gcc executable-monitor/test.c -o executable-monitor/test.out + gcc -DEXIT_WITH_MINUTES executable-monitor/test.c -o executable-monitor/test_exit_current_minutes.out + readlink -f executable-monitor/test.out + readlink -f executable-monitor/test_exit_current_minutes.out + exitStatus=$? + echo -e "::endgroup::" + if [ $exitStatus -eq 0 ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" + exit 1 + fi + + # Get future times from now, then look for that in the executable + - env: + stepName: Get Future Times + name: ${{ env.stepName }} + id: future-time + shell: bash + run: | + # ${{ env.stepName }} + set -x + echo -e "${{ env.bashInfo }} Getting Future Times to Compare Against ${{ env.bashEnd }}" + echo "exitCodeThreeMinutes=$(date --date='3 minutes' +%M)" >> "$GITHUB_ENV" + echo "successLineFourMinutes=$(date --date='4 minutes' +%H:%M)" >> "$GITHUB_ENV" + echo "successLineFiveMinutes=$(date --date='5 minutes' +%H:%M)" >> "$GITHUB_ENV" + echo "exitCodeSixMinutes=$(date --date='6 minutes' +%M)" >> "$GITHUB_ENV" + + - env: + stepName: "Functional | Exit Code | Exit Code Found" + name: ${{ env.stepName }} + id: exe-monitor-exit-code + uses: ./executable-monitor + with: + exe-path: executable-monitor/test.out + success-exit-code: 0 + log-dir: logDirectory + timeout-seconds: 60 + + - env: + stepName: "Functional | Success Line | Success Line Found" + name: ${{ env.stepName }} + id: exe-monitor-success-line + uses: ./executable-monitor + with: + exe-path: executable-monitor/test.out + success-line: "SLEEPING FOR 6 SECONDS" + log-dir: logDirectory + timeout-seconds: 30 + + - env: + stepName: "Functional | Exit Code and Success Line | Exit Code Found" + name: ${{ env.stepName }} + id: exe-monitor-find-exit-code + uses: ./executable-monitor + with: + exe-path: executable-monitor/test.out + success-line: "LINE_THAT_WILL_NOT_PRINT" + success-exit-code: 0 + timeout-seconds: 60 + + - env: + stepName: "Functional | Exit Code and Success Line | Success Line Found" + name: ${{ env.stepName }} + id: exe-monitor-find-success-line + uses: ./executable-monitor + with: + exe-path: executable-monitor/test.out + success-line: "SLEEPING FOR 6 SECONDS" + success-exit-code: 0 + timeout-seconds: 30 + + - env: + stepName: "Functional | Retry Needed | Exit Code Found" + name: ${{ env.stepName }} + id: exe-monitor-retry-find-exit-code + uses: ./executable-monitor + with: + exe-path: executable-monitor/test_exit_current_minutes.out + success-exit-code: ${{ env.exitCodeThreeMinutes }} + retry-attempts: 10 + timeout-seconds: 60 + + - env: + stepName: "Functional | Retry Needed | Success Line Found" + name: ${{ env.stepName }} + id: exe-monitor-retry-find-success-line + uses: ./executable-monitor + with: + exe-path: executable-monitor/test_exit_current_minutes.out + success-line: ${{ env.successLineFourMinutes }} + retry-attempts: 10 + timeout-seconds: 60 + + - env: + stepName: "Functional | Retry Needed | Exit Code and Success Line | Success Line Found" + name: ${{ env.stepName }} + id: exe-monitor-retry-both-inputs-find-success-line + uses: ./executable-monitor + with: + # Use the EXE that doesn't exit with the current minutes + exe-path: executable-monitor/test.out + success-line: ${{ env.successLineFiveMinutes }} + success-exit-code: 1 + retry-attempts: 10 + timeout-seconds: 60 + + - env: + stepName: "Functional | Retry Needed | Exit Code and Success Line | Exit Code Found" + name: ${{ env.stepName }} + id: exe-monitor-retry-both-inputs-find-exit + uses: ./executable-monitor + with: + exe-path: executable-monitor/test_exit_current_minutes.out + success-line: "LINE_THAT_WILL_NOT_PRINT" + success-exit-code: ${{ env.exitCodeSixMinutes }} + retry-attempts: 10 + timeout-seconds: 60 + + test-exe-monitor-failure-cases: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - env: + stepName: Install Windows Build tools + name: ${{ env.stepName }} + if: runner.os == 'Windows' + id: install-windows-build-tools + uses: microsoft/setup-msbuild@v1.1 + + - env: + stepName: Install Ubuntu Build Tools + name: ${{ env.stepName }} + if: runner.os == 'Linux' + id: install-ubuntu-build--tools + run: | + # Install Ubuntu Build Tools + echo -e "::group::${{ env.stepName }}" + sudo apt install build-essential + exitStatus=$? + echo -e "::endgroup::" + if [ $exitStatus -eq 0 ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" + exit 1 + fi + + - env: + stepName: Compile Executable Monitor Test Files + name: ${{ env.stepName }} + id: compile-executable-monitor-test-files + shell: bash + run: | + # ${{ env.stepName }} + echo -e "::group::${{ env.stepName }}" + gcc executable-monitor/test.c -o executable-monitor/test.out + gcc -DEXIT_WITH_MINUTES executable-monitor/test.c -o executable-monitor/test_exit_current_minutes.out + readlink -f executable-monitor/test.out + readlink -f executable-monitor/test_exit_current_minutes.out + exitStatus=$? + echo -e "::endgroup::" + if [ $exitStatus -eq 0 ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" + exit 1 + fi + + - env: + stepName: "API | Failure | No Executable Provided" + id: exe-monitor-fail-no-exe + uses: ./executable-monitor + continue-on-error: true + with: + timeout-seconds: 30 + + - env: + stepName: "API | Failure | No Success Condition Provided" + name: ${{ env.stepName }} + id: exe-monitor-fail-no-success-condition + uses: ./executable-monitor + continue-on-error: true + with: + exe-path: executable-monitor/test.out + log-dir: logDirectory + timeout-seconds: 30 + + - env: + stepName: "Functional | Failure | Timeout Cause No Success Line To Print" + name: ${{ env.stepName }} + id: exe-monitor-fail-timeout-no-success-line + uses: ./executable-monitor + continue-on-error: true + with: + exe-path: executable-monitor/test.out + # This is a line that would print if not for timeout + success-line: "SLEEPING FOR 9 SECONDS" + timeout-seconds: 2 + + - env: + stepName: "Functional | Failure | Timeout Cause No Exit Code" + name: ${{ env.stepName }} + id: exe-monitor-fail-timeout-no-exit-code + uses: ./executable-monitor + continue-on-error: true + with: + exe-path: executable-monitor/test.out + # This is an exit status that should be met if not for timeou + success-exit-code: 0 + timeout-seconds: 2 + + - env: + stepName: "Functional | Failure | Timeout Cause Neither Condition" + name: ${{ env.stepName }} + id: exe-monitor-fail-timeout-no-condition + uses: ./executable-monitor + continue-on-error: true + with: + exe-path: executable-monitor/test.out + # These are exit conditions that should be met if not for timeout + success-line: "SLEEPING FOR 9 SECONDS" + success-exit-code: 0 + timeout-seconds: 2 + + - env: + stepName: "Functional | Failure | Retries Timeout Cause No Success Line To Print" + name: ${{ env.stepName }} + id: exe-monitor-fail-retries-timeout-no-success-line + uses: ./executable-monitor + continue-on-error: true + with: + exe-path: executable-monitor/test.out + # This is a line that would print if not for timeout + success-line: "SLEEPING FOR 9 SECONDS" + timeout-seconds: 2 + retry-attempts: 2 + + - env: + stepName: "Functional | Failure | Retries Timeout Cause No Exit Code" + name: ${{ env.stepName }} + id: exe-monitor-fail-retries-no-exit-code + uses: ./executable-monitor + continue-on-error: true + with: + exe-path: executable-monitor/test.out + # This is an exit status that should be met if not for timeout + success-exit-code: 0 + timeout-seconds: 2 + retry-attempts: 2 + + - env: + stepName: "Functional | Failure | Retries Timeout Cause Neither Condition " + name: ${{ env.stepName }} + id: exe-monitor-fail-retries-no-success-condition + uses: ./executable-monitor + continue-on-error: true + with: + exe-path: executable-monitor/test.out + # These are exit conditions that should be met if not for timeout + success-line: "SLEEPING FOR 9 SECONDS" + success-exit-code: 0 + timeout-seconds: 2 + retry-attempts: 2 + + - env: + stepName: Check Failure Test Cases + name: ${{ env.stepName }} + id: check-failure-test-cases + shell: bash + run: | + # ${{ env.stepName }} + if [ "${{ steps.exe-monitor-fail-no-exe.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} | No Executable Provided | Failed As Intended ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} | No Executable Provided | Had Unexpected Pass ${{ env.bashEnd }}" + exit 1 + fi + + if [ "${{ steps.exe-monitor-fail-no-success-condition.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} | No Success Condition Provided | Failed As Intended ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} | No Success Condition Provided | Had Unexpected Pass ${{ env.bashEnd }}" + exit 1 + fi + + if [ "${{ steps.exe-monitor-fail-timeout-no-success-line.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} | Timeout Cause No Success Line To Print | Failed As Intended ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} | Timeout Cause No Success Line To Print | Had Unexpected Pass ${{ env.bashEnd }}" + exit 1 + fi + + if [ "${{ steps.exe-monitor-fail-timeout-no-exit-code.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} | Timeout Cause No Exit Code | Failed As Intended ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} | Timeout Cause No Exit Code | Had Unexpected Pass ${{ env.bashEnd }}" + exit 1 + fi + + if [ "${{ steps.exe-monitor-fail-timeout-no-condition.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} | Timeout Cause Neither Condition | Failed As Intended ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} | Timeout Cause Neither Condition | Had Unexpected Pass ${{ env.bashEnd }}" + exit 1 + fi + + if [ "${{ steps.exe-monitor-fail-retries-timeout-no-success-line.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} | Retries Timeout Cause No Success Line To Print | Failed As Intended ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} | Retries Timeout Cause No Success Line To Print | Had Unexpected Pass ${{ env.bashEnd }}" + exit 1 + fi + + if [ "${{ steps.exe-monitor-fail-retries-no-exit-code.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} | Retries Timeout Cause No Exit Code | Failed As Intended ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} | Retries Timeout Cause No Exit Code | Had Unexpected Pass ${{ env.bashEnd }}" + exit 1 + fi + + if [ "${{ steps.exe-monitor-fail-retries-no-success-condition.outcome}}" = "failure" ]; then + echo -e "${{ env.bashPass }} | Retries Timeout Cause Neither Condition | Failed As Intended ${{ env.bashEnd }}" + else + echo -e "${{ env.bashFail }} | Retries Timeout Cause Neither Condition | Had Unexpected Pass ${{ env.bashEnd }}" + exit 1 + fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b4c56dad..2756029e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,201 +31,6 @@ jobs: exclude-files: lexicon.txt exclude-dirs: build,docs - test-executable-monitor: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Compile executable monitor test - id: compile-executable-monitor-test - shell: bash - run: | - # Compile executable monitor test - echo "::group::Compile executable monitor test" - sudo apt install build-essential - if [ "$?" = "0" ]; then - echo -e "\033[32;3mInstalled build-essential\033[0m" - else - echo "::endgroup::" - echo -e "\033[32;31mInstall build-essential failed...\033[0m" - exit 1 - fi - gcc executable-monitor/test.c -o executable-monitor/test.out - gcc -DEXIT_WITH_MINUTES executable-monitor/test.c -o executable-monitor/test_exit_current_minutes.out - echo "::endgroup::" - if [ "$?" = "0" ]; then - readlink -f executable-monitor/test.out - echo -e "\033[32;3mCompiled executable-monitor/test.c\033[0m" - else - echo -e "\033[32;31mCompilation of executable-monitor/test.c failed...\033[0m" - exit 1 - fi - - - name: Functional Test | Success Case | Retries, Success Line, and Exit Code | Pass on Success Line - id: test-executable-monitor-action-success-line - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - timeout-seconds: 20 - success-line: "SLEEPING FOR 6 SECONDS" - success-exit-code: 0 - retry-attempts: 2 - - - name: Functional Test | Success Case | Retries, Success Line, and Exit Code | Pass on Exit Code - id: test-executable-monitor-action-exit-code - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - timeout-seconds: 75 - success-line: "LINE_THAT_WILL_NOT_PRINT_BUT_EXISTS" - success-exit-code: 0 - retry-attempts: 2 - - - name: Functional Test | Success Case | Retries, Success Line, and Exit Code, With log file - id: test-executable-monitor-action-log-file - uses: ./executable-monitor - with: - log-dir: demo_run_logs - exe-path: executable-monitor/test.out - timeout-seconds: 20 - success-line: "SLEEPING FOR 6 SECONDS" - success-exit-code: 0 - retry-attempts: 2 - - - name: Functional Test | Success Case | Retries, Success Line, No Exit Code - id: test-executable-monitor-action-no-exit-code - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - timeout-seconds: 20 - success-line: "SLEEPING FOR 6 SECONDS" - retry-attempts: 2 - - - name: Functional Test | Success Case | Retries, No Success Line, Exit Code - id: test-executable-monitor-action-no-success-line - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - timeout-seconds: 75 - success-exit-code: 0 - retry-attempts: 2 - - - name: Functional Test | Success Case | No Retries, Success Line, No Exit Code - id: test-executable-monitor-action-no-retry-attempts-success-line - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - success-line: "SLEEPING FOR 6 SECONDS" - timeout-seconds: 20 - - - name: Functional Test | Success Case | No Retries, No Success Line, Exit Code - id: test-executable-monitor-action-no-retry-attempts-exit-code - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - success-exit-code: 0 - timeout-seconds: 75 - - - name: API Test | Success Case | Retries, Success Line, No Exit Code, Use Default Timeout - id: test-executable-monitor-action-no-exit-code-no-timeout - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - success-line: "SLEEPING FOR 6 SECONDS" - retry-attempts: 2 - - - name: API Test | Success Case | Retries, No Success Line, Exit Code, Use Default Timeout - id: test-executable-monitor-action-no-success-line-no-timeout - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - success-exit-code: 0 - retry-attempts: 2 - - # Get future times from now, then look for that in the executable - - name: Get Future Times - id: future-time - run: | - # Get times for future demos - echo "time_success_line=$(date --date='2 minutes' +%H:%M)" >> "$GITHUB_ENV" - echo "time_exit_code=$(date --date='5 minutes' +%M)" >> "$GITHUB_ENV" - - - name: Functional Test | Sucess Case | Test Retry Run occurs on Wrong exit code - id: test-executable-monitor-action-retry-success-line - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - timeout-seconds: 20 - retry-attempts: 10 - success-line: ${{ env.time_success_line }} - - - name: Functional Test | Success Case | Retry On Wrong Exit Code - id: test-executable-monitor-action-retry-exit-code - uses: ./executable-monitor - with: - exe-path: executable-monitor/test_exit_current_minutes.out - success-exit-code: ${{ env.time_exit_code }} - timeout-seconds: 60 - retry-attempts: 10 - - - name: Functional Test | Failure Case | Fail On Timeout - id: test-executable-monitor-action-fail-on-timeout - continue-on-error: true - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - timeout-seconds: 1 - success-line: "SLEEPING FOR 12 SECONDS" - success-exit-code: 0 - retry-attempts: 2 - - - name: API Test | Failure Case | Retries, No Success Metric - continue-on-error: true - id: test-executable-monitor-API-no-success-metric - uses: ./executable-monitor - with: - exe-path: executable-monitor/test.out - timeout-seconds: 20 - retry-attempts: 2 - - - name: API Test | Failure Case | No Exe Path - continue-on-error: true - id: test-executable-monitor-API-no-exe-path - uses: ./executable-monitor - with: - timeout-seconds: 20 - success-line: "SLEEPING FOR 6 SECONDS" - success-exit-code: 0 - retry-attempts: 2 - - - name: Check Failure Test Cases - id: check-failure-test-cases - run: | - # Check Last Step Failed - echo "::group::Check Failure Test Cases" - if [ "${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}" = "failure" ]; then - echo -e "\033[32;3mFunctional Test | Failure Case | Fail On Timeout had outcome '${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}' as intended\033[0m" - else - echo "::endgroup::" - echo -e "\033[32;31mFunctional Test | Failure Case | Fail On Timeout had unexpected '${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}' exit condition\033[0m" - exit 1 - fi - if [ "${{ steps.test-executable-monitor-API-no-success-metric.outcome}}" = "failure" ]; then - echo -e "\033[32;3mAPI Test | Failure Case | Retries, No Success Metric had outcome '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' as intended\033[0m" - else - echo "::endgroup::" - echo -e "\033[32;31mAPI Test | Failure Case | Retries, No Success Metric had unexpected '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' exit condition\033[0m" - exit 1 - fi - if [ "${{ steps.test-executable-monitor-API-no-exe-path.outcome}}" = "failure" ]; then - echo -e "\033[32;3mAPI Test | Failure Case | No Exe Path had outcome '${{ steps.test-executable-monitor-API-no-exe-path.outcome}}' as intended\033[0m" - else - echo "::endgroup::" - echo -e "\033[32;31mAPI Test | Failure Case | No Exe Path had unexpected '${{ steps.test-executable-monitor-API-no-exe-path.outcome}}' exit condition\033[0m" - exit 1 - fi - echo "::endgroup::" - echo -e "All failure tests failed correctly" - test-complexity-check: runs-on: ubuntu-latest steps: diff --git a/executable-monitor/README.md b/executable-monitor/README.md index 81ee26ad..c68c8bce 100644 --- a/executable-monitor/README.md +++ b/executable-monitor/README.md @@ -1,8 +1,30 @@ -Test the case where exit status code and success line are given but hit timeout -python3 executable-monitor.py --log-dir . --success-exit-status 0 --success-line "Sleeping for 27 seconds" --retry-attempts 2 --timeout-seconds 10 --exe-path test.out ; echo $? +# Executable Monitor Summary: +This is a python program designed to run an executable and monitor it. +It can either look for a string being printed to the program's standard out, or check for an exit condition. +It supports retry logic, as well as deadlocking programs. +The tests for the executable monitor mostly live inside of the CI-CD-Github-Actions/.github/workflows/test.yml file. +A few tests have been added below for ease of copying and pasting it though. +These tests are meant to be run using the test.c file that lives inside of this directory. +test.c prints out the system time and then sleeps for a set duration +# Provided Test.c: + test.c is a simple C program that prints the current minutes, hours, and seconds. + As well as how long the program is going to sleep for. + These values can then be used as a test for the executable-monitor. +```gcc test.c -o test.out``` -# Run a test where we find the success line but the exe does not exit, ensure exit status is 0 -#python3 executable-monitor.py --log-dir . --success-line "Sleeping for 27 seconds" --retry-attempts 2 --timeout-seconds 10 --exe-path test.out; echo $? +Test.c can also be set to exit with the current time in minutes by using the command. +By setting the exit code or success line to be a time in the future a test can be written that requires retry logic. +By setting the success-line to be a time in the future the program will need to "retry" until this time occurs +By setting the exit code of the program to be the current minute, the program will need to "retry" until this exit code is seen. +```gcc test.c -DEXIT_WITH_MINUTES -o test.out``` -Test a run where the thread will timeout while waiting for the success message, ensure exit status is 1 -# python3 executable-monitor.py --log-dir . --success-line "Sleeping for 27 seconds" --retry-attempts 2 --timeout-seconds 2 --exe-path test.out; echo $? \ No newline at end of file + +# How to call the program locally with some basic tests. + Success Test | Test the case where the success line is found but the exe does not exit, ensure exit status is 0 +```python3 executable-monitor.py --success-line "SLEEPING FOR 6 SECONDS" --retry-attempts 2 --timeout-seconds 10 --exe-path test.out; echo $?``` + +Success Test | Test the case where the success line is not found, but the exe exits, ensure exit status is 0 +```python3 executable-monitor.py --success-line "THIS WILL NEVER PRINT" --success-exit-code 0 --retry-attempts 2 --timeout-seconds 60 --exe-path test.out; echo $?``` + +Failure Test | Test the case where exit status code and success line are given but hit timeout, ensure exit status is 1 +```python3 executable-monitor.py --success-exit-code 0 --success-line "SLEEPING FOR 12 SECONDS" --retry-attempts 2 --timeout-seconds 10 --exe-path test.out ; echo $?``` diff --git a/executable-monitor/action.yml b/executable-monitor/action.yml index 59cedba4..467fb8ea 100644 --- a/executable-monitor/action.yml +++ b/executable-monitor/action.yml @@ -7,15 +7,13 @@ inputs: log-dir: description: 'Path to directory to store logs.' required: false - default: "" success-line: description: 'Line of output from executable indicating success.' required: false - default: "" timeout-seconds: - description: 'Maximum amount of time to run the executable. Default is 600.' + description: 'Maximum amount of time to run the executable. Default is 180.' required: false - default: 300 + default: 180 retry-attempts: description: 'Number of times to re-launch the binary to check for success.' required: false @@ -23,82 +21,87 @@ inputs: success-exit-code: description: 'Exit status that indicates that the executable completed successfully. Required if --success-line is not used.' required: false - default: "" runs: using: "composite" - steps: - - name: Install Dependencies + steps: + - env: + # The bash escape character is \033 + # At time of writing, you can't add a global environment to an + # action so stuck with this. If this gets changed please move this + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + stepName: Install Dependencies + name: ${{ env.stepName }} + shell: bash run: | - # Install Dependencies - echo "::group::Install Dependencies" + # ${{ env.stepName }} + echo -e "::group::Install Dependencies" pip install -r $GITHUB_ACTION_PATH/requirements.txt - echo "::endgroup::" - if [ "$?" = "0" ]; then - echo -e "\033[32;3mInstalled all dependencies\033[0m" - exit 0 - else - echo -e "\033[32;31mDependencies Install failed...\033[0m" - exit 1 - fi + exitStatus=$? + echo -e "::endgroup::" + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" + + - env: + bashPass: \033[32;1mPASSED - + bashInfo: \033[33;1mINFO - + bashFail: \033[31;1mFAILED - + bashEnd: \033[0m + stepName: Run Executable with Monitoring + name: ${{ env.stepName }} shell: bash - - name: Run Executable with Monitoring run: | # Run Executable with Monitoring - echo "::group::Argument Parsing" - optArgs=" " + echo -e "::group::${{ env.bashInfo }} ${{ env.stepName }} ${{ env.bashEnd }}" + # Make sure we have an exit condition to look for if [ "${{ inputs.success-exit-code }}" = "" ] && [ "${{ inputs.success-line }}" = "" ]; then - echo "::endgroup::" - echo -e "\033[32;31mDid not supply an input of success-line or success-exit-code to search for\033[0m" + echo -e "::endgroup::\n${{ env.bashFail}} Did not supply an input of success-line or success-exit-code to search for ${{ env.bashEnd }}" exit 1 fi + # Initial Arguments + args="$GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }}" + args+=" --timeout-seconds=${{ inputs.timeout-seconds }}" + # Check if an exit code was provided - if [ "${{ inputs.success-exit-code }}" != "" ]; then - optArgs="--success-exit-code=${{ inputs.success-exit-code }} $optArgs" - echo "Adding --success-exit-code=${{ inputs.success-exit-code }} to the call" + if [ -n "${{ inputs.success-exit-code }}" ]; then + args+=" --success-exit-code=${{ inputs.success-exit-code }}" fi # Check for log directory/if a log file should be created - if [ "${{ inputs.log-dir }}" != "" ]; then - optArgs="--log-dir=${{ inputs.log-dir}} $optArgs" - echo "Adding --log-dir=${{ inputs.log-dir}} to the call" + if [ -n "${{ inputs.log-dir }}" ]; then + args+=" --log-dir=${{ inputs.log-dir}}" fi # Check for retry attempts - if [ "${{ inputs.retry-attempts }}" != "" ]; then - optArgs="--retry-attempts=${{ inputs.retry-attempts }} $optArgs" - echo "Adding --retry-attempts=${{ inputs.retry-attempts }} to the call" + if [ -n "${{ inputs.retry-attempts }}" ]; then + args+=" --retry-attempts=${{ inputs.retry-attempts }}" fi - # Check if a success line was provided. Converting the github input to a bash variable - # Will cause it to tokenize on spaces when passed into python - # So just only add the success line if it exits. + # set +e so if the run fails we can capture that and print custom error message set +e - if [ "${{ inputs.success-line }}" != "" ]; then - successLineArgs="--success-line=\"${{ inputs.success-line}}\" $optArgs" - echo "Adding --success-line=${{ inputs.success-line}} to the call" - echo "::endgroup::" - echo -e "\033[1;33mRunning Command: python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} --success-line="${{ inputs.success-line }}" $optArgs \033[0m" - echo "::group::Executable Output" - python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} $optArgs --success-line="${{ inputs.success-line}}" + + # Check for success line to search for + if [ -n "${{ inputs.success-line }}" ]; then + echo -e "${{ env.bashInfo }} Running: python3 ${args} --success-line=${{ inputs.success-line}} ${{ env.bashEnd }}" + python3 ${args} --success-line="${{ inputs.success-line}}" else - echo "::endgroup::" - echo -e "\033[1;33mRunning Command: python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} $optArgs \033[0m" - echo "::group::Executable Output" - python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} $optArgs + echo -e "${{ env.bashInfo }} Running: python3 ${args} ${{ env.bashEnd }}" + python3 ${args} fi + # Store status as the echo group will overwrite it exitStatus=$? - echo "::endgroup::" - if [ "$exitStatus" = "0" ]; then - echo -e "\033[32;3mValid exit status found\033[0m" - exit 0 + set -e + echo -e "::endgroup::" + + if [ $exitStatus -eq 0 ]; then + echo -e "${{ env.bashPass }} ${{ env.stepName }} ${{ env.bashEnd }}" else - echo -e "\033[32;31mDid not find a valid exit condition\033[0m" - set -e - exit 1 + echo -e "${{ env.bashFail }} ${{ env.stepName }} ${{ env.bashEnd }}" fi - shell: bash + exit $exitStatus diff --git a/executable-monitor/executable-monitor.py b/executable-monitor/executable-monitor.py index e3a85f6a..deca7af2 100755 --- a/executable-monitor/executable-monitor.py +++ b/executable-monitor/executable-monitor.py @@ -7,16 +7,26 @@ import logging from multiprocessing import Process -def runAndMonitor(args): - # Set up logging - logging.getLogger().setLevel(logging.NOTSET) +# Set up logging +logging.getLogger().setLevel(logging.NOTSET) + +# This script is meant for FreeRTOS PR checks, which all run using bash shells +# So wrap the important lines in bash escaped colours. +bashPass="\033[32;1m" +bashWarn="\033[33;1m" +bashFail="\033[31;1m" +bashEnd="\033[0m" +# Add stdout handler to logging +stdout_logging_handler = logging.StreamHandler(sys.stdout) +stdout_logging_handler.setLevel(logging.DEBUG) +stdout_logging_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') +stdout_logging_handler.setFormatter(stdout_logging_formatter) +logging.getLogger().addHandler(stdout_logging_handler) - # Add stdout handler to logging - stdout_logging_handler = logging.StreamHandler(sys.stdout) - stdout_logging_handler.setLevel(logging.DEBUG) - stdout_logging_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - stdout_logging_handler.setFormatter(stdout_logging_formatter) - logging.getLogger().addHandler(stdout_logging_handler) +def runAndMonitor(args): + # On Windows we need to add the child's logging to the global logger + if os.name == 'nt': + logging.getLogger().addHandler(stdout_logging_handler) exe_abs_path = os.path.abspath(args.exe_path) if args.log_dir is not None: @@ -69,14 +79,15 @@ def runAndMonitor(args): exe.kill() time.sleep(.05) else: - logging.info(exe_stdout_line) + # Making an assumption that device output is adding a \n already + logging.info(exe_stdout_line.rstrip("\r\n")) # Check for timeout cur_time_seconds = time.time() if cur_time_seconds >= timeout_time_seconds: logging.info(f"TIMEOUT OF {args.timeout_seconds} SECONDS HIT") exit_condition_met = True - + # Sleep for a short duration between loops to not steal all system resources # time.sleep(.05) @@ -86,10 +97,13 @@ def runAndMonitor(args): logging.info(f"PARSING REST OF LOG") # Capture remaining output and check for the successful line for exe_stdout_line in exe.stdout.readlines(): - logging.info(exe_stdout_line) + # Making an assumption that device output is adding a \n already + logging.info(exe_stdout_line.rstrip("\r\n")) if args.success_line is not None and args.success_line in exe_stdout_line: success_line_found = True - logging.info(f"SUCCESS_LINE_FOUND: {exe_stdout_line}") + # Insert a blank line to make the log look better + print() + logging.info(f"{bashPass}SUCCESS_LINE_FOUND: {exe_stdout_line}{bashEnd}") logging.info("END OF DEVICE OUTPUT") logging.info("EXECUTABLE RUN SUMMARY:") @@ -97,41 +111,31 @@ def runAndMonitor(args): exit_status = 1 # Check if a success line was found if that is an option if ( args.success_line is not None) and (not success_line_found ): - logging.error("Success Line: Success line not output.") + logging.error(f"{bashFail}Success Line: Success line not output.{bashEnd}") exit_status = 1 elif( args.success_line is not None) and ( success_line_found ): exit_status = 0 logging.info(f"Success Line: Success line was output") - + # Check if a exit code was found if that was an option if ( ( exit_status != 0 ) and ( args.success_exit_code is not None) ): # If the executable had to be force killed mark it as a failure if( not exe_exitted): - logging.error("Exit Code: Executable did not exit by itself.\n") + logging.error(f"{bashFail}Exit Code: Executable did not exit by itself.{bashEnd}\n") exit_status = 1 # If the executable exited with a different status mark it as a failure elif ( ( exe_exitted ) and ( exe_exit_status != args.success_exit_code ) ): - logging.error(f"Exit Code: {exe_exit_status} is not equal to requested exit code of {args.success_exit_code}\n") + logging.error(f"{bashFail}Exit Code: {exe_exit_status} is not equal to requested exit code of {args.success_exit_code}{bashEnd}\n") exit_status = 1 # If the executable exited with the same code as requested mark a success elif ( ( exe_exitted ) and ( exe_exit_status == args.success_exit_code ) ): - logging.info(f"Exit Code: Executable exited with requested exit code") + logging.info(f"{bashPass}Exit Code: Executable exited with requested exit code{bashEnd}") exit_status = 0 - logging.info(f"Runner thread exiting with status {exit_status}\n") + logging.info(f"Runner thread exiting with status {exit_status}") exit(exit_status) if __name__ == '__main__': - # Set up logging - logging.getLogger().setLevel(logging.NOTSET) - - # Add stdout handler to logging - stdout_logging_handler = logging.StreamHandler(sys.stdout) - stdout_logging_handler.setLevel(logging.DEBUG) - stdout_logging_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') - stdout_logging_handler.setFormatter(stdout_logging_formatter) - logging.getLogger().addHandler(stdout_logging_handler) - # Parse arguments parser = ArgumentParser(description='Executable monitor.') parser.add_argument('--exe-path', @@ -162,15 +166,15 @@ def runAndMonitor(args): args = parser.parse_args() if args.success_exit_code is None and args.success_line is None: - logging.error("Must specify at least one of the following: --success-line, --success-exit-code.") + logging.error(f"{bashFail}Must specify at least one of the following: --success-line, --success-exit-code.{bashEnd}") sys.exit(1) elif args.success_exit_code is not None and args.success_line is not None: - logging.warning("Received an option for success-line and success-exit-code.") - logging.warning("Be aware: This program will report SUCCESS on either of these conditions being met") + logging.warning(f"{bashWarn}Received an option for success-line and success-exit-code.{bashEnd}") + logging.warning(f"{bashWarn}Be aware: This program will report SUCCESS on either of these conditions being met{bashEnd}") if not os.path.exists(args.exe_path): - logging.error(f'Input executable path \"{args.exe_path}\" does not exist.') + logging.error(f"{bashFail}Input executable path \"{args.exe_path}\" does not exist.{bashEnd}") sys.exit(1) # Convert any relative path (like './') in passed argument to absolute path. @@ -194,31 +198,31 @@ def runAndMonitor(args): logging.info(f"Running executable: {exe_abs_path} ") logging.info(f"Timeout (seconds) per run: {args.timeout_seconds}") - + if not args.retry_attempts: args.retry_attempts = 0 else: logging.info(f"Will relaunch the executable {args.retry_attempts} times to look for a valid success metric") - + if args.success_line is not None: logging.info(f"Searching for success line: {args.success_line}") if args.success_exit_code is not None: logging.info(f"Searching for exit code: {args.success_exit_code}") - + # Small increase on the timeout to allow the thread to try and timeout threadTimeout = ( args.timeout_seconds + 3 ) for attempts in range(0,args.retry_attempts + 1): exit_status = 1 - # Set the timeout for the thread + # Set the timeout for the thread thread = Process(target=runAndMonitor, args=(args,)) thread.start() - # Wait for the thread to join, or hit a timeout. + # Wait for the thread to join, or hit a timeout. thread.join(timeout=threadTimeout) # As join() always returns None, you must call is_alive() after join() to decide whether a timeout happened - # If the thread is still alive, the join() call timed out. + # If the thread is still alive, the join() call timed out. if ( ( thread.exitcode is None ) and ( thread.is_alive() ) ): # Print the thread timeout they passed in to the log - logging.warning(f"EXECUTABLE HAS HIT TIMEOUT OF {threadTimeout - 3} SECONDS: FORCE KILLING THREAD") + logging.warning(f"{bashWarn}EXECUTABLE HAS HIT TIMEOUT OF {threadTimeout - 3} SECONDS: FORCE KILLING THREAD{bashEnd}") thread.kill() exit_status = 1 else: @@ -226,10 +230,10 @@ def runAndMonitor(args): logging.info(f"THREAD EXITED WITH EXITCODE {exit_status}") if( ( attempts < args.retry_attempts ) and exit_status != 0 ): - logging.warning(f"DID NOT RECEIVE SUCCESSFUL EXIT STATUS, TRYING RE-ATTEMPT {attempts+1} OF {args.retry_attempts}\n") + logging.warning(f"{bashWarn}DID NOT RECEIVE SUCCESSFUL EXIT STATUS, TRYING RE-ATTEMPT {attempts+1} OF {args.retry_attempts}{bashEnd}\n") else: break - logging.info(f"EXECUTABLE MONITOR EXITING WITH STATUS: {exit_status}") - # Report final exit status if no successful run occured + logging.warning(f"{bashWarn}EXECUTABLE MONITOR EXITING WITH STATUS: {exit_status}{bashEnd}") + # Report Final Exit Status sys.exit(exit_status) diff --git a/executable-monitor/test.c b/executable-monitor/test.c index 564e17f0..27a5df1d 100644 --- a/executable-monitor/test.c +++ b/executable-monitor/test.c @@ -1,30 +1,55 @@ -#include #include #include #include -#include +#include - -typedef struct DateAndTime +typedef struct TimeStruct { - int year; - int month; - int day; - int hour; - int minutes; - int seconds; - int msec; -} DateAndTime; + uint64_t hour; + uint64_t minutes; + uint64_t seconds; + uint64_t msec; +} TimeStruct; + + +#if defined( WIN32 ) || defined( _WIN32 ) || defined( __WIN32__ ) || defined( __NT__ ) || defined( WIN64 ) || defined( __WIN64 ) + #include + /* Remove the warning about implicit sleep even with windows.h included */ + extern void sleep( int miliseconds ); + void getTime( struct TimeStruct * currentTime ) + { + SYSTEMTIME st, lt; + + GetLocalTime( < ); + currentTime->hour = lt.wHour; + currentTime->minutes = lt.wMinute; + currentTime->seconds = lt.wSecond; + currentTime->msec = lt.wMilliseconds; + } +#else /* if defined( WIN32 ) || defined( _WIN32 ) || defined( __WIN32__ ) || defined( __NT__ ) || defined( WIN64 ) || defined( __WIN64 ) */ + #include + #include + void getTime( struct TimeStruct * currentTime ) + { + struct timeval tv; + struct tm * tm; + + gettimeofday( &tv, NULL ); + tm = localtime( &tv.tv_sec ); + currentTime->hour = tm->tm_hour; + currentTime->minutes = tm->tm_min; + currentTime->seconds = tm->tm_sec; + currentTime->msec = ( int ) ( tv.tv_usec / 1000 ); + } +#endif /* if defined( WIN32 ) || defined( _WIN32 ) || defined( __WIN32__ ) || defined( __NT__ ) || defined( WIN64 ) || defined( __WIN64 ) */ int main( int argc, char ** argv ) { - DateAndTime date_and_time; - struct timeval tv; - struct tm * tm; + TimeStruct currentTime = { 0 }; int32_t loop = 0; - int32_t totalLoops = 5; + int32_t totalLoops = 5U; int32_t exitCode = 0; if( argc == 1 ) @@ -37,49 +62,33 @@ int main( int argc, if( argc == 2 ) { - totalLoops = ( int32_t ) atoi( argv[ 2 ] ); + totalLoops = atoi( argv[ 1 ] ); printf( "Will run for requested %d loops\n", totalLoops ); } if( argc == 3 ) { - exitCode = atoi( argv[ 3 ] ); + exitCode = atoi( argv[ 2 ] ); printf( "Will exit with supplied exit code %d\n", exitCode ); } setvbuf( stdout, NULL, _IONBF, 0 ); - for(int i = 1; i < totalLoops; i++) + for( int i = 1U; i < totalLoops; i++ ) { - gettimeofday( &tv, NULL ); - tm = localtime( &tv.tv_sec ); - /* Add 1900 to get the right year value */ - /* read the manual page for localtime() */ - /* date_and_time.year = tm->tm_year + 1900; */ - /* Months are 0 based in struct tm */ - date_and_time.year = tm->tm_year + 1900; - date_and_time.month = tm->tm_mon + 1; - date_and_time.day = tm->tm_mday; - date_and_time.hour = tm->tm_hour; - date_and_time.minutes = tm->tm_min; - date_and_time.seconds = tm->tm_sec; - date_and_time.msec = ( int ) ( tv.tv_usec / 1000 ); - - fprintf( stdout, "%02d:%02d:%02d.%03d %02d-%02d-%04d TEST APPLICIATION SLEEPING FOR %d SECONDS\n", - date_and_time.hour, - date_and_time.minutes, - date_and_time.seconds, - date_and_time.msec, - date_and_time.day, - date_and_time.month, - date_and_time.year, - i * 3U - ); + getTime( ¤tTime ); + printf( "%02llu:%02llu:%02llu.%03llu TEST APPLICATION SLEEPING FOR %d SECONDS\n", + currentTime.hour, + currentTime.minutes, + currentTime.seconds, + currentTime.msec, + i * 3U ); sleep( i * 3U ); } -#ifdef EXIT_WITH_MINUTES - exitCode = date_and_time.minutes; -#endif - printf( "EXITING TEST APPLICICATION WITH EXIT CODE = %d\n",exitCode ); + + #ifdef EXIT_WITH_MINUTES + exitCode = currentTime.minutes; + #endif + printf( "EXITING TEST APPLICATION WITH EXIT CODE = %d\n", exitCode ); return exitCode; }