Logging update (#64)

* Executable monitor update to use a thread to run and monitor the output. Parent task then monitors the join time of the child to monitor if it has timed out. Also adding a different check for if the exe exits, but we haven't found a desired success message * Use echo groups to clean output * Adding some tests to the executable monitor, making it so that if it finds the success message or the exit status it works * Swapping from exit status to exit code * Adding default timeout tests, remove failure on no timeout provided test * Run the happy path test once where it passes with the success line, and once where it passes with the timeout * Changes to the action.yml to remake log file optional ---------
FreeRTOS · Jul 21, 2023 · a2ca3c7 · a2ca3c7
1 parent 133f201
commit a2ca3c7
Show file tree

Hide file tree

Showing 6 changed files with 487 additions and 141 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -23,6 +23,175 @@ jobs:
           path: coreMQTT
           exclude-files: lexicon.txt
           exclude-dirs: build,docs
+
+  test-executable-monitor:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Compile executable monitor test
+        id: compile-executable-monitor-test
+        shell: bash
+        run: |
+          # Compile executable monitor test
+          echo "::group::Compile executable monitor test"
+          sudo apt install build-essential
+          if [ "$?" = "0" ]; then
+            echo -e "\033[32;3mInstalled build-essential\033[0m"
+          else
+            echo "::endgroup::"
+            echo -e "\033[32;31mInstall build-essential failed...\033[0m"
+            exit 1
+          fi
+          gcc executable-monitor/test.c -o executable-monitor/test.out
+          echo "::endgroup::"
+          if [ "$?" = "0" ]; then
+            readlink -f executable-monitor/test.out
+            echo -e "\033[32;3mCompiled executable-monitor/test.c\033[0m"
+          else
+            echo -e "\033[32;31mCompilation of executable-monitor/test.c failed...\033[0m"
+            exit 1
+          fi
+
+      - name: Functional Test | Success Case | Executable Monitor | Retries, Success Line, and Exit Code | Pass on Success Line
+        id: test-executable-monitor-action-success-line
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          timeout-seconds: 20
+          success-line: "SLEEPING FOR 6 SECONDS"
+          success-exit-code: 0
+          retry-attempts: 2
+
+      - name: Functional Test | Success Case | Executable Monitor | Retries, Success Line, and Exit Code | Pass on Exit Code
+        id: test-executable-monitor-action-exit-code
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          timeout-seconds: 75
+          success-line: "LINE_THAT_WILL_NOT_PRINT_BUT_EXISTS"
+          success-exit-code: 0
+          retry-attempts: 2
+
+      - name: Functional Test | Success Case | Executable Monitor | Retries, Success Line, and Exit Code, With log file
+        id: test-executable-monitor-action-log-file
+        uses: ./executable-monitor
+        with:
+          log-dir: demo_run_logs
+          exe-path: executable-monitor/test.out
+          timeout-seconds: 20
+          success-line: "SLEEPING FOR 6 SECONDS"
+          success-exit-code: 0
+          retry-attempts: 2
+
+      - name: Functional Test | Success Case | Executable Monitor | Retries, Success Line, No Exit Code
+        id: test-executable-monitor-action-no-exit-code
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          timeout-seconds: 20
+          success-line: "SLEEPING FOR 6 SECONDS"
+          retry-attempts: 2
+
+      - name: Functional Test | Success Case | Executable Monitor | Retries, No Success Line, Exit Code
+        id: test-executable-monitor-action-no-success-line
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          timeout-seconds: 75
+          success-exit-code: 0
+          retry-attempts: 2
+
+      - name: Functional Test | Success Case | Executable Monitor | No Retries, Success Line, No Exit Code
+        id: test-executable-monitor-action-no-retry-attempts-success-line
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          success-line: "SLEEPING FOR 6 SECONDS"
+          timeout-seconds: 20
+
+      - name: Functional Test | Success Case | Executable Monitor | No Retries, No Success Line, Exit Code
+        id: test-executable-monitor-action-no-retry-attempts-exit-code
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          success-exit-code: 0
+          timeout-seconds: 75
+
+      - name: API Test | Success Case | Executable Monitor | Retries, Success Line, No Exit Code, Use Default Timeout
+        id: test-executable-monitor-action-no-exit-code-no-timeout
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          success-line: "SLEEPING FOR 6 SECONDS"
+          retry-attempts: 2
+
+      - name: API Test | Success Case | Executable Monitor | Retries, No Success Line, Exit Code, Use Default Timeout
+        id: test-executable-monitor-action-no-success-line-no-timeout
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          success-exit-code: 0
+          retry-attempts: 2
+
+      - name: Functional Test | Failure Case | Executable Monitor | Fail On Timeout
+        id: test-executable-monitor-action-fail-on-timeout
+        continue-on-error: true
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          timeout-seconds: 1
+          success-line: "SLEEPING FOR 12 SECONDS"
+          success-exit-code: 0
+          retry-attempts: 2
+
+      - name: API Test | Failure Case | Executable Monitor | Retries, No Success Line, No Exit Code
+        continue-on-error: true
+        id: test-executable-monitor-API-no-success-metric
+        uses: ./executable-monitor
+        with:
+          exe-path: executable-monitor/test.out
+          timeout-seconds: 20
+          retry-attempts: 2
+
+      - name: API Test | Failure Case | Executable Monitor | No Exe Path
+        continue-on-error: true
+        id: test-executable-monitor-API-no-exe-path
+        uses: ./executable-monitor
+        with:
+          timeout-seconds: 20
+          success-line: "SLEEPING FOR 6 SECONDS"
+          success-exit-code: 0
+          retry-attempts: 2
+
+      - name: Check Failure Test Cases
+        id: check-failure-test-cases
+        run: |
+          # Check Last Step Failed
+          echo "::group::Check Failure Test Cases"
+          if [ "${{ steps.test-executable-monitor-API-no-success-metric.outcome}}" = "failure" ]; then
+            echo -e "\033[32;3mtest-executable-monitor-API-no-success-metric had outcome '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' as intended\033[0m"
+          else
+            echo "::endgroup::"
+            echo -e "\033[32;31mtest-executable-monitor-API-no-success-metric had unexpected '${{ steps.test-executable-monitor-API-no-success-metric.outcome}}' exit condition\033[0m"
+            exit 1
+          fi
+          if [ "${{ steps.test-executable-monitor-API-no-exe-path.outcome}}" = "failure" ]; then
+          echo -e "\033[32;3mtest-executable-monitor-API-no-exe-path had outcome '${{ steps.test-executable-monitor-API-no-exe-path.outcome}}' as intended\033[0m"
+          else
+            echo "::endgroup::"
+            echo -e "\033[32;31mtest-executable-monitor-API-no-exe-path had unexpected '${{ steps.test-executable-monitor-API-no-exe-path.outcome}}' exit condition\033[0m"
+            exit 1
+          fi
+          if [ "${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}" = "failure" ]; then
+          echo -e "\033[32;3mtest-executable-monitor-action-fail-on-timeout had outcome '${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}' as intended\033[0m"
+          else
+            echo "::endgroup::"
+            echo -e "\033[32;31mtest-executable-monitor-action-fail-on-timeout had unexpected '${{ steps.test-executable-monitor-action-fail-on-timeout.outcome}}' exit condition\033[0m"
+            exit 1
+          fi
+          echo "::endgroup::"
+          echo -e "All failure tests failed correctly"
+
   test-complexity-check:
     runs-on: ubuntu-latest
     steps:

diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+*.out
+logDir/
+demo_run_logs/
diff --git a/executable-monitor/README.md b/executable-monitor/README.md
@@ -0,0 +1,8 @@
+Test the case where exit status code and success line are given but hit timeout
+python3 executable-monitor.py --log-dir . --success-exit-status 0 --success-line "Sleeping for 27 seconds"  --retry-attempts 2 --timeout-seconds 10 --exe-path test.out ; echo $?
+
+# Run a test where we find the success line but the exe does not exit, ensure exit status is 0
+#python3 executable-monitor.py --log-dir . --success-line "Sleeping for 27 seconds"  --retry-attempts 2 --timeout-seconds 10 --exe-path test.out; echo $?
+
+Test a run where the thread will timeout while waiting for the success message, ensure exit status is 1
+# python3 executable-monitor.py --log-dir . --success-line "Sleeping for 27 seconds"  --retry-attempts 2 --timeout-seconds 2 --exe-path test.out; echo $?
diff --git a/executable-monitor/action.yml b/executable-monitor/action.yml
@@ -6,11 +6,12 @@ inputs:
     required: true
   log-dir:
     description: 'Path to directory to store logs.'
-    required: true
+    required: false
+    default: ""
   success-line:
     description: 'Line of output from executable indicating success.'
     required: false
-    default: "Demo completed successfully."
+    default: ""
   timeout-seconds:
     description: 'Maximum amount of time to run the executable. Default is 600.'
     required: false
@@ -19,14 +20,57 @@ inputs:
     description: 'Number of times to re-launch the binary to check for success.'
     required: false
     default: 0
+  success-exit-code:
+    description: 'Exit status that indicates that the executable completed successfully. Required if --success-line is not used.'
+    required: false
+    default: ""
 
 runs:
   using: "composite"
   steps: 
-      - name: Install dependencies
-        run: pip install -r $GITHUB_ACTION_PATH/requirements.txt
+      - name: Install Dependencies
+        run: |
+          # Install Dependencies
+          echo "::group::Install Dependencies"
+          pip install -r $GITHUB_ACTION_PATH/requirements.txt
+          echo "::endgroup::"
+          if [ "$?" = "0" ]; then
+            echo -e "\033[32;3mInstalled all dependencies\033[0m"
+            exit 0
+          else
+            echo -e "\033[32;31mDependencies Install failed...\033[0m"
+            exit 1
+          fi
         shell: bash
-      - name: Run executable with monitoring script
+      - name: Run Executable with Monitoring
         run: |
-          python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} --success-line="${{ inputs.success-line }}" --log-dir=${{ inputs.log-dir }} --retry-attempts=${{ inputs.retry-attempts }}
+          # Run Executable with Monitoring
+          echo "::group::Executable Output"
+          optArgs=" " 
+          if [ "${{ inputs.success-exit-code }}" = "" ] && [ "${{ inputs.success-line }}" = "" ]; then
+            echo "::endgroup::"
+            echo -e "\033[32;31mDid not supply an input of success-line or success-exit-code to search for\033[0m"
+            exit 1
+          fi
+          if [ "${{ inputs.success-exit-code }}" != "" ]; then
+             optArgs="--success-exit-code=${{ inputs.success-exit-code }}$optArgs "
+          fi
+          
+          if [ "${{ inputs.log-dir }}" != "" ]; then
+            optArgs="--log-dir=${{ inputs.log-dir}} $optArgs"
+          fi
+          echo "optArgs= $optArgs"
+          if [ "${{ inputs.success-line }}" != "" ]; then
+            python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} $optArgs --success-line="${{ inputs.success-line}}"
+          else
+            python3 $GITHUB_ACTION_PATH/executable-monitor.py --exe-path=${{ inputs.exe-path }} --timeout-seconds=${{ inputs.timeout-seconds }} $optArgs
+          fi
+          echo "::endgroup::"
+          if [ "$?" = "0" ]; then
+            echo -e "\033[32;3mValid exit status found\033[0m"
+            exit 0
+          else
+            echo -e "\033[32;31mDid not find a valid exit condition\033[0m"
+            exit 1
+          fi
         shell: bash