test(api): install opentrons and simulate (#17048)

.github/workflows/api-test-lint-deploy.yaml

@@ -73,8 +73,6 @@ jobs:
     strategy:
       matrix:
         os: ['windows-2022', 'ubuntu-22.04', 'macos-latest']
-        # TODO(mc, 2022-02-24): expand this matrix to 3.8 and 3.9,
-        # preferably in a nightly cronjob on edge or something
         python: ['3.10']
         with-ot-hardware: ['true', 'false']
         exclude:
@@ -128,6 +126,60 @@ jobs:
           files: ./api/coverage.xml
           flags: api
 
+  test-package:
+    name: 'installed package tests on ${{ matrix.os }}'
+    timeout-minutes: 5
+    strategy:
+      matrix:
+        os: ['ubuntu-22.04', 'macos-latest', 'windows-2022']
+    runs-on: '${{ matrix.os }}'
+    steps:
+      - uses: 'actions/checkout@v4'
+      - name: 'Fix actions/checkout odd handling of tags'
+        if: startsWith(github.ref, 'refs/tags')
+        run: |
+          git fetch -f origin ${{ github.ref }}:${{ github.ref }}
+          git checkout ${{ github.ref }}
+      - uses: 'actions/setup-python@v4'
+        with:
+          python-version: '3.10'
+      - name: Set up package-testing
+        id: setup
+        if: ${{ matrix.os != 'windows-2022' }}
+        working-directory: package-testing
+        shell: bash
+        run: make setup
+      - name: Set up package-testing (Windows)
+        id: setup-windows
+        if: ${{ matrix.os == 'windows-2022' }}
+        working-directory: package-testing
+        shell: pwsh
+        run: make setup-windows
+      - name: Run the tests
+        if: ${{ matrix.os != 'windows-2022' }}
+        shell: bash
+        id: test
+        working-directory: package-testing
+        run: make test
+      - name: Run the tests (Windows)
+        shell: pwsh
+        id: test-windows
+        working-directory: package-testing
+        run: make test-windows
+      - name: Save the test results
+        if: ${{ always() && steps.setup.outcome == 'success' || steps.setup-windows.outcome == 'success' }}
+        id: results
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-test-results-${{ matrix.os }}
+          path: package-testing/results
+      - name: Set job summary
+        if: ${{ always() }}
+        run: |
+          echo "## Opentrons Package Test Results ${{matrix.os}}" >> $GITHUB_STEP_SUMMARY
+          echo "### Test Outcome: Unixy ${{ steps.test.outcome }} Windows: ${{ steps.test-windows.outcome }}" >> $GITHUB_STEP_SUMMARY
+          echo "[Download the test results artifact](${{steps.results.outputs.artifact-url}})" >> $GITHUB_STEP_SUMMARY
+
   deploy:
     name: 'deploy opentrons package'
     needs: [test]

.gitignore

@@ -163,3 +163,5 @@ opentrons-robot-app.tar.gz
 mock_dir
 .npm-cache/
 .eslintcache
+
+package-testing/results

package-testing/Makefile

@@ -0,0 +1,43 @@
+VENV_DIR ?= venv
+RESULTS_DIR ?= results
+TESTS ?= all
+
+.PHONY: setup
+setup:
+	@echo "Setting up environment for Unix-like system..."
+	./setup.sh $(VENV_DIR)
+
+.PHONY:  setup-windows
+setup-windows:
+	@echo "Setting up environment for Windows..."
+	pwsh -ExecutionPolicy Bypass -File ./setup.ps1 $(VENV_DIR)
+
+.PHONY: clean
+clean:
+	@echo "Removing the results directory $(RESULTS_DIR)..."
+	rm -rf $(RESULTS_DIR) || true
+
+.PHONY: clean-windows
+clean-windows:
+	@echo "Removing the results directory $(RESULTS_DIR)..."
+	pwsh -Command "if (Test-Path '$(RESULTS_DIR)') { Remove-Item -Recurse -Force '$(RESULTS_DIR)' }"
+
+.PHONY: teardown
+teardown: clean
+	rm -rf $(VENV_DIR) || true
+
+.PHONY: teardown-windows
+teardown-windows: clean-windows
+		pwsh -Command "if (Test-Path '$(VENV_DIR)') { Remove-Item -Recurse -Force '$(VENV_DIR)' }"
+
+
+
+.PHONY: test
+test: clean
+	@echo "Running $(TESTS) tests for Unix-like system..."
+	python run_tests.py $(VENV_DIR) $(RESULTS_DIR) $(TESTS)
+
+.PHONY: test-windows
+test-windows: clean-windows
+	@echo "Running $(TESTS) tests for Windows..."
+	python run_tests.py $(VENV_DIR) $(RESULTS_DIR) $(TESTS)

package-testing/README.md

@@ -0,0 +1,34 @@
+# Test Scripts for the opentrons package
+
+## Structure
+
+- Makefile has targets for setting up, tearing down, and running tests for windows and unix-ish systems
+- setup.\* is the script run create the virtual environment and install the packages
+- help.\* is a script to test --help
+- simulate.\* is a script to test that the simulation runs and produces the expected status code
+- run_tests.py is the main script that drives test execution and contains the test mapping data
+
+## Use the tests on Linux and Mac
+
+1. cd package-testing
+2. pyenv local 3.10
+3. make setup - note that this deletes and recreates the virtual environment
+4. make test
+
+## Use the tests on Windows
+
+- powershell is mapped to pwsh and is version 7
+- python is on the path is version 3.10.\*
+
+1. cd package-testing
+2. make setup-windows - note that this deletes and recreates the virtual environment
+3. make test-windows
+
+## Notes
+
+- find . -name "\*.sh" -exec shellcheck {} +
+
+## TODO
+
+- setup shellcheck and python linting
+- more tests

package-testing/help.ps1

@@ -0,0 +1,69 @@
+#!/usr/bin/env pwsh
+
+<#
+.SYNOPSIS
+Validate `opentrons_simulate --help`.
+
+.PARAMETER TestKey
+The test key to identify the test.
+
+.PARAMETER ExpectedOutput
+The expected string to validate in the output of `opentrons_simulate --help`.
+
+.PARAMETER Venv
+The virtual environment directory (default: "venv").
+
+.PARAMETER ResultDir
+The result directory to store logs (default: "results").
+#>
+
+param (
+    [string]$TestKey,
+    [string]$ExpectedOutput,
+    [string]$Venv = "venv",
+    [string]$ResultDir = "results"
+)
+
+# Ensure the result directory exists
+if (-not (Test-Path -Path $ResultDir)) {
+    New-Item -ItemType Directory -Path $ResultDir | Out-Null
+}
+
+$resultFile = Join-Path -Path $ResultDir -ChildPath "$TestKey.txt"
+
+Write-Output "Activating virtual environment $Venv..."
+$venvActivate = Join-Path -Path $Venv -ChildPath "Scripts/Activate.ps1"
+if (-not (Test-Path -Path $venvActivate)) {
+    Write-Error "FAIL: Virtual environment not found at $venv"
+    exit 1
+}
+
+# Source the virtual environment
+& $venvActivate
+
+Write-Output "Validating opentrons_simulate --help for test: $TestKey..."
+
+# Run the command and capture the output and return code
+$output = & opentrons_simulate --help 2>&1
+$returnCode = $LASTEXITCODE
+
+if ($returnCode -ne 0) {
+    Write-Output "FAIL: Return code is $returnCode, expected 0" | Tee-Object -FilePath $resultFile
+    Write-Output "Output was:" | Add-Content -Path $resultFile
+    $output | Add-Content -Path $resultFile
+    Rename-Item -Path $resultFile -NewName "${resultFile.Substring(0, $resultFile.Length - 4)}_FAIL.txt"
+    exit 1
+}
+
+Write-Output "PASS: Return code is $returnCode, expected 0" | Tee-Object -FilePath $resultFile
+Write-Output "Output was:" | Add-Content -Path $resultFile
+$output | Add-Content -Path $resultFile
+
+if ($output -match [regex]::Escape($ExpectedOutput)) {
+    Write-Output "PASS: Output contains expected string" | Tee-Object -FilePath $resultFile -Append
+    Write-Output "PASS: Test $TestKey completed successfully." | Tee-Object -FilePath $resultFile -Append
+    exit 0
+}
+
+Write-Output "FAIL: Output does not contain expected string" | Tee-Object -FilePath $resultFile -Append
+exit 1

package-testing/help.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Function to validate `opentrons_simulate --help`
+# Arguments:
+#   1. Test Key (required)
+#   2. Virtual Environment Directory (optional, default: "venv")
+#   3. Result Directory (optional, default: "results")
+test_opentrons_simulate_help() {
+    local test_key="$1"
+    local expected_output="${2}"
+    local venv="${3:-"venv"}"
+    local result_dir="${4:-"results"}"
+    mkdir -p "$result_dir"
+    local result_file="$result_dir/$test_key.txt"
+
+    echo "Activating virtual environment $venv..."
+    # shellcheck disable=SC1091
+    source "$venv/bin/activate"
+
+    echo "Validating opentrons_simulate --help for test: $test_key..."
+
+    local output
+    local return_code
+    output=$(opentrons_simulate --help 2>&1)
+    return_code=$?
+
+    if [ $return_code -ne 0 ]; then
+        echo "FAIL: Return code is $return_code, expected 0" | tee "$result_file"
+        echo "Output was:" >> "$result_file"
+        echo "$output" >> "$result_file"
+        mv "$result_file" "${result_file%.txt}_FAIL.txt"
+        return 1
+    fi
+
+    echo "PASS: Return code is $return_code, expected 0" | tee "$result_file"
+    echo "Output was:" >> "$result_file"
+    echo "$output" >> "$result_file"
+
+    if echo "$output" | grep -q "$expected_output"; then
+        echo "PASS: Output contains expected string" | tee -a "$result_file"
+        echo "PASS: Test $test_key completed successfully." | tee -a "$result_file"
+        return 0
+    fi
+    echo "FAIL: Output does not contain expected string" | tee -a "$result_file"
+    return 1
+}
+
+test_opentrons_simulate_help "$@"