Refactor card.binary HIL GitHub workflow. (#144)

- Remove the unused localtunnel code. If we ever need to switch to localtunne from tunnelmole, we can pull it out of the git history. - Remove usage of third party action pyTooling/Actions/with-post-step. Instead, to run cleanup steps, use the `if: always()` construct. - Remove usage of third party action JarvusInnovations/background-action@v1. The scripts we need to run in the background can simply be launched in the background using & syntax. - Refactor a bunch of conditional logic that was hard to read. Instead of using if statements, we were chaining together a bunch of different Bash statements using logical operators. Using if statements is simpler to read and understand what exactly is being checked. - Refactor run_tunnelmole.sh.
blues · Mar 25, 2024 · a4f8c65 · a4f8c65
1 parent bd9556e
commit a4f8c65
Show file tree

Hide file tree

Showing 3 changed files with 125 additions and 153 deletions.
diff --git a/.github/workflows/notecard-binary-tests.yml b/.github/workflows/notecard-binary-tests.yml
@@ -41,39 +41,24 @@ jobs:
         # START_MD5SRV set to false to skip starting the MD5 server. There should be one
         # already running locally with MD5SRV_PORT/ADDRESS/TOKEN set correspondingly.
         START_MD5SRV: true
-        # START_LOCALTUNNEL, set to false to skip starting the localtunnel.
-        START_LOCALTUNNEL: false
         # START_TUNNELMOLE: set to false to skip starting tunnel mole.
         START_TUNNELMOLE: true
         # When neither tunneling solution is used (because they're already instantiated outside of the workflow)
         # be sure to set MD5SRV_URL in the environment
     steps:
-      - name: Checkout
+      - name: Checkout note-c repo
         uses: actions/checkout@v3
+
       - name: Generate MD5 Server Token
         run: |
           [ -n "$MD5SRV_TOKEN" ] || echo "MD5SRV_TOKEN=`uuidgen`" >> $GITHUB_ENV
           # propagate the environment variable so that it's available in the `env` context
           echo "MD5SRV_PORT=$MD5SRV_PORT" >> $GITHUB_ENV
-      - name: Check Env Vars
+
+      - name: Check env vars
         run: |
           . scripts/check_runner_config.sh
           echo NOTEHUB_PROXY_ROUTE_ALIAS=$NOTEHUB_PROXY_ROUTE_ALIAS
-      - name: Prep MD5 Server
-        uses: pyTooling/Actions/[email protected]
-        with:
-          main: |
-            [ -e md5srv-files ] || mkdir md5srv-files
-            rm -rf md5srv-files/*
-            md5url=http://${MD5SRV_ADDRESS}:${MD5SRV_PORT}/
-          post: |
-            # the JarvusInnovations/background-action@v1 that launches the background
-            # process doesn't clean them up. We do that here. MD5SRV_PID is set in the
-            # ./scripts/run_md5srv.sh script
-            echo Stop MD5 Server
-            [ -n "$MD5SRV_PID" ] || (echo "MD5SRV_PID not set" && exit 1)
-            [ -z "$MD5SRV_PID" ] || kill $MD5SRV_PID
-            rm -rf md5srv-files
 
       - name: Install PlatformIO dependencies
         if: env.FLASH_TEST_FIRMWARE!='false'
@@ -90,33 +75,12 @@ jobs:
           # copy only files in note-c
           find "$GITHUB_WORKSPACE" -maxdepth 1 -type f -exec cp "{}" "${NOTE_C_DEP}" \;
 
-      - name: Start MD5 Server
-        uses: JarvusInnovations/background-action@v1
-        with:
-          run: |
-            ./scripts/run_md5srv.sh
-          wait-on:
-            # just a dummy wait-on since this is required.
-            file:${{github.workspace}}/scripts/run_md5srv.sh
-
-      # When done this way, the background process is terminated at the end of the step,
-      # At least when running with `act`. The same may be true of github runners also.
-      # - name: Start MD5 Server
-      #   uses: pyTooling/Actions/[email protected]
-      #   with:
-      #     main: |
-      #       ./run_md5srv.sh
-      #       echo "MD5SRV_PID=$MD5SRV_PID"
-      #       echo "MD5SRV_PID=$MD5SRV_PID" >> $GITHUB_ENV
-      #       echo "writing server log to `realpath md5srv.log`"
-      #       md5url=http://${MD5SRV_ADDRESS}:${MD5SRV_PORT}/
-      #     post: |
-      #       echo Stop MD5 Server
-      #       [ -n "$MD5SRV_PID" ] || (echo "MD5SRV_PID not set" && exit 1)
-      #       # [ -z "$MD5SRV_PID" ] || kill $MD5SRV_PID
-      #       rm -rf md5srv-files
-
-      - name: Build and Upload Test Firmware
+      - name: Start MD5 server
+        run: |
+          mkdir md5srv-files
+          ./scripts/run_md5srv.sh
+
+      - name: Build and upload test firmware
         if: env.FLASH_TEST_FIRMWARE!='false'
         run: |
           source venv/bin/activate
@@ -126,118 +90,82 @@ jobs:
           platformio test -e debug --without-testing --upload-port "$STLINK_PROGRAMMER_PORT" --project-dir "$PIO_PROJECT_DIR"
           timeout 10 ./scripts/wait_for_file.sh "$SWAN_SERIAL"
 
-      - name: Start localtunnel
-        if: env.START_LOCALTUNNEL!='false'
-        id: localtunnel
-        uses: Rei-x/expose-localtunnel-action@main
-        with:
-          ports: ${{ env.MD5SRV_PORT }}
-      - name: Fetch localtunnel URL
-        if: env.START_LOCALTUNNEL!='false'
-        run: |
-          MD5SRV_URL="${{steps.localtunnel.outputs.url-1}}"
-          echo "MD5SRV_URL=$MD5SRV_URL" >> $GITHUB_ENV
-
-      - name: Prep tunnelmole
-        if: env.START_TUNNELMOLE!='false'
-        uses: pyTooling/Actions/[email protected]
-        with:
-          main: |
-            [ ! -e tmole.log ] || rm tmole.log
-            sleep 2 # otherwise it thinks we exited early
-          post: |
-            # the JarvusInnovations/background-action@v1 that launches the background
-            # process doesn't clean them up. We do that here. TMOLE_PID is set in the
-            # ./scripts/run_tunnelmole.sh script
-            echo Stop Tunnelmole
-            [ -n "$TMOLE_PID" ] || (echo "TMOLE_PID not set" && exit 1)
-            [ -z "$TMOLE_PID" ] || kill $TMOLE_PID
-            # Remove the tmole_ready file, which may be leftover from a prior
-            # run.
-            rm -f $GITHUB_WORKSPACE/tmole_ready
-
       - name: Start tunnelmole
-        uses: JarvusInnovations/background-action@v1
         if: env.START_TUNNELMOLE!='false'
-        with:
-          run: |
-            ./scripts/run_tunnelmole.sh
-          log-output-if: true
-          wait-on:
-            file:${{github.workspace}}/tmole_ready
+        run: |
+          rm -f tmole.log
+          ./scripts/run_tunnelmole.sh
 
-      - name: Check server is available
+      - name: Check MD5 server is available
         run: |
           # the request will return a 401 from md5srv, but that's expected without the access token
           # Curl still returns success because it could contact the server
           code=`curl -s -o /dev/null -w "%{http_code}" $MD5SRV_URL`
-          [ "$code" -lt "500" ] || ( echo "5xx error ($code) from tunnel." && exit 1 )
+          if [ "$code" -ge "500" ]; then
+            echo "5xx error ($code) from tunnel."
+            exit 1
+          else
+            echo "MD5 server is available."
+          fi
 
       - name: Create Notehub accesss token
         if: env.CREATE_NOTEHUB_ROUTES!='false'
         run: |
-            curl -f -X POST \
-                -L 'https://${{env.NOTEHUB}}/oauth2/token' \
-                -H 'content-type: application/x-www-form-urlencoded' \
-                -d grant_type=client_credentials \
-                -d client_id=$NOTEHUB_CLIENT_ID \
-                -d client_secret=$NOTEHUB_CLIENT_SECRET | \
-                { token=$(jq -r .access_token); echo "NOTEHUB_ACCESS_TOKEN=$token" >> $GITHUB_ENV; }
-
-      - name: Create Notehub HTTP Route
+          curl -f -X POST \
+            -L 'https://${{env.NOTEHUB}}/oauth2/token' \
+            -H 'content-type: application/x-www-form-urlencoded' \
+            -d grant_type=client_credentials \
+            -d client_id=$NOTEHUB_CLIENT_ID \
+            -d client_secret=$NOTEHUB_CLIENT_SECRET | \
+            { token=$(jq -r .access_token); echo "NOTEHUB_ACCESS_TOKEN=$token" >> $GITHUB_ENV; }
+
+      - name: Create Notehub HTTP route
         if: env.CREATE_NOTEHUB_ROUTES!='false'
-        uses: pyTooling/Actions/[email protected]
-        with:
-          main: |
-            # ?note=1 instructs the MD5 server to process the content as an event, extracting the path
-            # from the event body.
-            route_req=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_HTTP_ROUTE_LABEL" --arg URL "$MD5SRV_URL/?note=1" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
-              '{ "label":$LABEL, "type":"http", "http":{ "timeout":$TIMEOUT, "filter": { "type":"include", "files": ["cardbinary.qo"] }, "url":$URL, "http_headers": { "X-Access-Token":$TOKEN } } }'`
-            echo $route_req
-            route=`echo "$route_req" | curl -s -f -X POST -L "https://$NOTEHUB_API/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
-                    -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
-            echo $route
-            route_uid=`echo $route | jq -r .uid`
-            [ -n "$route_uid" ]
+        run: |
+          # ?note=1 instructs the MD5 server to process the content as an event, extracting the path
+          # from the event body.
+          route_req=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_HTTP_ROUTE_LABEL" --arg URL "$MD5SRV_URL/?note=1" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
+            '{ "label":$LABEL, "type":"http", "http":{ "timeout":$TIMEOUT, "filter": { "type":"include", "files": ["cardbinary.qo"] }, "url":$URL, "http_headers": { "X-Access-Token":$TOKEN } } }'`
+          echo "Notehub HTTP route request: $route_req"
+          route=`echo "$route_req" | curl -s -f -X POST -L "https://$NOTEHUB_API/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
+                  -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
+          echo "Notehub HTTP route: $route"
+          route_uid=`echo $route | jq -r .uid`
+          if [ -n "$route_uid" ]; then
             echo "NOTEHUB_HTTP_ROUTE_UID=$route_uid" >> $GITHUB_ENV
-          post: |
-            echo Delete Notehub HTTP Route
-            [ "$DELETE_NOTEHUB_ROUTES" == "false" ] || ([ -n "$NOTEHUB_HTTP_ROUTE_UID" ] && curl -f -s -X DELETE \
-                -L "https://$NOTEHUB_API/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_HTTP_ROUTE_UID" \
-                -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN")
+          else
+            echo "Failed to create or parse Notehub HTTP route."
+            exit 1
+          fi
 
-      - name: Create Notehub Proxy Route
-        if: env.CREATE_NOTEHUB_ROUTES!='false'
-        uses: pyTooling/Actions/[email protected]
-        with:
-          main: |
-            ALIAS=$NOTEHUB_PROXY_ROUTE_ALIAS
-            route=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_PROXY_ROUTE_LABEL" --arg URL "$MD5SRV_URL" --arg ALIAS "$ALIAS" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
-              '{ "label":$LABEL, "type":"proxy", "proxy":{ "timeout":$TIMEOUT, "url":$URL, "alias":$ALIAS, "http_headers": { "X-Access-Token":$TOKEN } } }' \
-              | curl -s -f -X POST -L "https://api.notefile.net/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
-                    -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
-            echo $route
-            route_uid=`echo $route | jq -r .uid`
-            [ -n $route_uid ]
+      - name: Create Notehub proxy route
+        run: |
+          ALIAS="$NOTEHUB_PROXY_ROUTE_ALIAS"
+          route=`jq -n --arg TOKEN "$MD5SRV_TOKEN" --arg LABEL "$NOTEHUB_PROXY_ROUTE_LABEL" --arg URL "$MD5SRV_URL" --arg ALIAS "$ALIAS" --argjson TIMEOUT $NOTEHUB_ROUTE_TIMEOUT \
+            '{ "label":$LABEL, "type":"proxy", "proxy":{ "timeout":$TIMEOUT, "url":$URL, "alias":$ALIAS, "http_headers": { "X-Access-Token":$TOKEN } } }' \
+            | curl -s -f -X POST -L "https://api.notefile.net/v1/projects/${NOTEHUB_PROJECT_UID}/routes" \
+                  -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN" -d @-`
+          echo "Notehub proxy route: $route"
+          route_uid=`echo "$route" | jq -r .uid`
+          if [ -n $route_uid ]; then
             echo "NOTEHUB_PROXY_ROUTE_UID=$route_uid" >> $GITHUB_ENV
             echo "NOTEHUB_PROXY_ROUTE_ALIAS=$ALIAS" >> $GITHUB_ENV
-          post: |
-            echo Delete Notehub Proxy Route
-            [ "$DELETE_NOTEHUB_ROUTES" == "false" ] || ([ -n "$NOTEHUB_PROXY_ROUTE_UID" ] && curl -f -s -X DELETE \
-                -L "https://api.notefile.net/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_PROXY_ROUTE_UID" \
-                -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN")
+          else
+            echo "Failed to create or parse Notehub proxy route."
+            exit 1
+          fi
 
-      - name: Run Tests
+      - name: Run tests
         run: |
           source venv/bin/activate
           cd $PIO_PROJECT_DIR
           platformio test -v -e debug \
              --without-building --without-uploading \
              --test-port "$SWAN_SERIAL" \
              --json-output-path test.json \
-             --junit-output-path test.xml \
+             --junit-output-path test.xml
 
-      - name: Publish Test Report
+      - name: Publish test report
         uses: mikepenz/action-junit-report@v3
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -246,3 +174,48 @@ jobs:
           report_paths: '**/test/hitl/card.binary/test*.xml'
           check_name: Notecard Binary HIL Tests
           require_tests: true
+
+      - name: Cleanup Notehub proxy route
+        if: always()
+        run: |
+          if [ "$DELETE_NOTEHUB_ROUTES" == "true" && -n "$NOTEHUB_PROXY_ROUTE_UID" ]; then
+            echo "Deleting Notehub proxy route."
+            curl -f -s -X DELETE \
+              -L "https://api.notefile.net/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_PROXY_ROUTE_UID" \
+              -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN"
+          fi
+
+      - name: Cleanup Notehub HTTP route
+        if: always()
+        run: |
+          if [ "$DELETE_NOTEHUB_ROUTES" == "true" ] && [ -n "$NOTEHUB_HTTP_ROUTE_UID" ]; then
+            echo "Deleting Notehub HTTP route."
+            curl -f -s -X DELETE \
+              -L "https://$NOTEHUB_API/v1/projects/$NOTEHUB_PROJECT_UID/routes/$NOTEHUB_HTTP_ROUTE_UID" \
+              -H "Authorization: Bearer $NOTEHUB_ACCESS_TOKEN"
+          fi
+
+
+      - name: Cleanup tunnelmole
+        if: always()
+        run: |
+          if [ -n "$TMOLE_PID" ]; then
+            echo "Stopping tunnelmole."
+            kill $TMOLE_PID
+          else
+            echo "Tunnelmole not running (TMOLE_PID is empty)."
+          fi
+
+      - name: Cleanup MD5 server
+        if: always()
+        run: |
+          if [ -d md5srv-files ]; then
+            echo "Deleting md5srv-files directory."
+            rm -rf md5srv-files
+          fi
+          if [ -n "$MD5SRV_PID" ]; then
+            echo "Stopping MD5 server."
+            kill $MD5SRV_PID
+          else
+            echo "MD5 server not running (MD5SRV_PID is empty)."
+          fi
diff --git a/scripts/run_md5srv.sh b/scripts/run_md5srv.sh
@@ -2,6 +2,3 @@
 python3 ./test/hitl/scripts/md5srv.py --dir md5srv-files --save > md5srv.log 2>&1 &
 MD5SRV_PID=$!
 echo "MD5SRV_PID=$MD5SRV_PID" >> $GITHUB_ENV
-
-# Prevent the action from thinking we exited early
-sleep 2
diff --git a/scripts/run_tunnelmole.sh b/scripts/run_tunnelmole.sh
@@ -25,26 +25,28 @@ fi
 
 sleep 1
 
-# find the url
-exec 3< tmole.log
-# ignore the first line
-read <&3 output
-read <&3 output
-
-echo "Getting MD5 server URL from tmole.log..." >&2
-MD5SRV_URL=`echo "$output" | grep https | cut -d " " -f1`
+
+TIMEOUT=8
+SECONDS=0
+MD5SRV_URL=""
+# Check tmole.log for the MD5 server URL every second.
+until [ "$SECONDS" -ge "$TIMEOUT" ]
+do
+    MD5SRV_URL=$(grep -oP "^https://[\w\d\.-]+" tmole.log)
+
+    if [ -n "$MD5SRV_URL" ]; then
+        break
+    else
+        SECONDS=$((SECONDS+1))
+        sleep 1
+    fi
+done
+
 if [ -z "$MD5SRV_URL" ]; then
-    echo "ERROR: Failed to get MD5 server URL." >&2
+    echo "ERROR: Timed out waiting for MD5 server URL to get written to tmole.log." >&2
     exit 1
-else
-    echo "INFO: Got MD5 server URL." >&2
 fi
 
+echo "INFO: Got MD5 server URL from tmole.log." >&2
 echo "MD5SRV_URL=$MD5SRV_URL" >> $GITHUB_ENV
-# Only create tmole_ready once MD5SRV_URL has been set, as the next step in the
-# GitHub workflow depends on it.
-touch tmole_ready
-
-sleep 2
-
 echo "INFO: tmole ready. Logging to `realpath tmole.log`" >&2