AUTH-73 feat(rtp): example RTP for testing (#14703)

## Overview
<https://opentrons.atlassian.net/browse/AUTH-73>
<https://opentrons.atlassian.net/browse/RDEVOPS-71>

~~Examples only.~~
~~This PR will not integrate these examples into the analyses
snapshot.~~
### I lied.

---------

Co-authored-by: Derek Maggio <[email protected]>

app-testing/.gitignore

@@ -1,3 +1,5 @@
 .env
 results
 analysis_results/*.json
+files/generated_protocols/*
+!files/generated_protocols/.keepme

app-testing/Makefile

@@ -8,26 +8,26 @@ black-check:
 
 .PHONY: ruff
 ruff:
-	python -m pipenv run python -m ruff . --fix --unsafe-fixes
+	python -m pipenv run python -m ruff check . --fix
 
 .PHONY: ruff-check
 ruff-check:
-	python -m pipenv run python -m ruff .
+	python -m pipenv run python -m ruff check .
 
 .PHONY: mypy
 mypy:
 	python -m pipenv run python -m mypy conftest.py automation tests citools
 
 .PHONY: lint
-lint:
-	$(MAKE) black-check
-	$(MAKE) ruff-check
-	$(MAKE) mypy
+lint: black-check ruff-check mypy
 
 .PHONY: format
-format:
+format: 
+	@echo runnning black
 	$(MAKE) black
+	@echo running ruff
 	$(MAKE) ruff
+	@echo formatting the readme with yarn prettier
 	$(MAKE) format-readme
 
 .PHONY: test-ci
@@ -50,10 +50,6 @@ teardown:
 format-readme:
 	yarn prettier --ignore-path .eslintignore --write app-testing/**/*.md
 
-.PHONY: print-protocols
-print-protocols:
-	python -m pipenv run python print_protocols.py
-
 .PHONY: install-pipenv
 install-pipenv:
 	python -m pip install -U pipenv
@@ -67,9 +63,15 @@ snapshot-test-update:
 	python -m pipenv run pytest -k analyses_snapshot_test --snapshot-update
 
 TARGET ?= edge
+CACHEBUST := $(shell date +%s)
 
 .PHONY: build-opentrons-analysis
 build-opentrons-analysis:
 	@echo "Building docker image for $(TARGET)"
-	@echo "If you want to build a different version, run 'make build-docker TARGET=<version>'"
-	docker build --build-arg OPENTRONS_VERSION=$(TARGET) -t opentrons-analysis:$(TARGET) citools/.
+	@echo "If you want to build a different version, run 'make build-opentrons-analysis TARGET=<version>'"
+	@echo "Cache is always busted to ensure latest version of the code is used"
+	docker build --build-arg OPENTRONS_VERSION=$(TARGET) --build-arg CACHEBUST=$(CACHEBUST) -t opentrons-analysis:$(TARGET) citools/.
+
+.PHONY: generate-protocols
+generate-protocols:
+	python -m pipenv run python -m automation.data.protocol_registry

app-testing/Pipfile

@@ -4,23 +4,19 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [packages]
-pytest = "==7.4.3"
-black = "==23.11.0"
-selenium = "==4.15.2"
-importlib-metadata = "==6.8.0"
+pytest = "==8.1.1"
+black = "==24.3.0"
+selenium = "==4.19.0"
+importlib-metadata = "==7.1.0"
 requests = "==2.31.0"
-python-dotenv = "==1.0.0"
-pytest-xdist = "==3.5.0"
-mypy = "==1.7.1"
-types-requests = "==2.31.0.10"
-rich = "==13.7.0"
-atomicwrites = "==1.4.1"
-pyreadline3 = "==3.4.1"
-pydantic = "==2.5.2"
-pygithub = "==2.1.1"
-ruff = "==0.1.6"
-docker = "==6.1.3"
-syrupy = "==4.6.0"
+python-dotenv = "==1.0.1"
+mypy = "==1.9.0"
+types-requests = "==2.31.0.20240311"
+rich = "==13.7.1"
+pydantic = "==2.6.4"
+ruff = "==0.3.4"
+docker = "==7.0.0"
+syrupy = "==4.6.1"
 pytest-html = "==4.1.1"
 
 [requires]

app-testing/README.md

@@ -68,35 +68,35 @@ pipenv run python -i locators.py
 - Only have 1 robot connected at once.
   - Build locators like you have more than 1 to future proof.
 
-## Analysis Test
+### Analyses Snapshot Test
 
-The analysis test `pipenv run pytest -k test_analyses` is driven by the comma delimited string variable `APP_ANALYSIS_TEST_PROTOCOLS` in `.env`
-This allows us to run one or many.
+> The primary test in this module.
 
-### Adding protocols to the analysis test
+The analyses snapshot test runs protocol analysis using `TARGET` branch or tag then compares them against snapshots found on `TEST_SOURCE` branch or tag.
 
-1. add the protocol file named according to the naming convention in the files/protocols appropriate folder
-1. add the protocol stem to `protocol_files.py`
-1. add the protocol data as a property to `protocols.py`
-1. run `make print-protocols`
+#### Protocol Files Location
 
-### Analyses Snapshot Test
+The set of protocols to analyze is defined inside of `app-testing/.env` file, under the `APP_ANALYSIS_TEST_PROTOCOLS` and `APP_ANALYSIS_TEST_PROTOCOLS_WITH_OVERRIDES` variables.
 
-The analyses snapshot test runs protocol analysis using `TARGET` branch or tag then compares them against snapshots found on `TEST_SOURCE` branch or tag.
+#### Protocol Files with Overrides
 
-#### Protocol Files Location
+Sometimes we want to have a bunch of protocols that are just slightly different from each other. This is especially helpful with negative test cases. We can have a protocol that depending on the value of a variable does different things. You may then override the variable to test different scenarios.
 
-The set of protocols to analyze is defined inside of `app-testing/.env` file, under the `APP_ANALYSIS_TEST_PROTOCOLS` variable. These protocols must exist inside of `app-testing/files/protocols` folder.
+The best way to learn this is by example. Look at:
 
-- Protocol Designer protocols go in the `json` folder
-- Python protocols go in the `python` folder.
+- `app-testing/files/protocolsFlex_X_v2_18_NO_PIPETTES_Overrides_BadTypesInRTP.py`
+- `app-testing/automation/data/protocols_with_overrides.py`
+- `make generate-protocols`
+- see the protocols generated in `app-testing/files/generated_protocols/`
 
 #### Analysis Snapshots Location
 
-Analysis snapshots are located inside of `app-testing/tests/__snapshots__/analyses_snapshot_test` folder.
+Analysis snapshots are located inside of `app-testing/tests/__snapshots__/analyses_snapshot_test` folder. These are generated. If you want to update them, see below.
 
 #### Running Analysis Snapshot Tests Locally
 
+> Note: Passing `TARGET` can be done as below or in the `.env` file.
+
 To run analysis snapshot tests locally, you must first build the Docker image by running the following command:
 
 ```bash
@@ -135,6 +135,6 @@ Given the scenario that you want to see if the latest version of `chore_release-
 - If you want to compare against the previous release branch, then TEST_SOURCE is chore_release-v7.1.0.
 - If you want to compare your in-progress release branch against the previous release branch, then TEST_SOURCE is `<your-branch-name>`.
 
-run the Workflow Dispatch job
+##### Run the Workflow Dispatch job
 
 - `gh workflow run 'Analyses Snapshot Test' --ref chore_release-v7.2.0 -f TARGET=chore_release-v7.2.0 -f TEST_SOURCE=chore_release-v7.1.0`

app-testing/automation/data/protocol.py

@@ -1,22 +1,24 @@
 """Model of a protocol for testing."""
+
 import hashlib
 import os
 from pathlib import Path
 from typing import Literal, Optional
 
 from pydantic import BaseModel, Field
 
-from automation.data.protocol_files import names
 from automation.resources.robot_data import module_types
 
+GENERATED_PROTOCOLS_FOLDER = "generated_protocols"
+OVERRIDE_MONIKER = "_Override_"
+
 
 class Protocol(BaseModel):
     """Model to describe a protocol used in a test."""
 
-    file_name: names = Field(description="file name not including extension")
+    file_stem: str = Field(description="file name not including extension")
     file_extension: Literal["json", "py"] = Field(description="file extension of the protocol")
-    protocol_name: str = Field(description="the protocol name which will appear in the protocol name field in the app")
-    robot: Literal["OT-2", "Flex"] = Field(description="the robot type which will appear in the robot field in the app")
+    robot: Literal["OT2", "Flex"] = Field(description="the robot type which will appear in the robot field in the app")
     app_error: bool = Field(description="will analysis with the app raise an error")
     robot_error: bool = Field(description="will analysis with the robot raise an error")
     app_analysis_error: Optional[str] = Field(description="the exact error shown in the app popout", default=None)
@@ -26,17 +28,27 @@ class Protocol(BaseModel):
     modules: Optional[list[module_types]] = Field(description="list of modules that will show in the app", default=None)
     description: Optional[str] = Field(description="Details about this protocol", default=None)
     expected_test_failure: bool = Field(description="Is this test expected to fail", default=False)
-    expected_test_reason: Optional[str] = Field(description="Reason test is failing", default=False)
+    expected_test_reason: Optional[str] = Field(description="Reason test is failing", default=None)
+    override_variable_name: Optional[str] = Field(description="The variable name to override", default=None)
+    override_value: Optional[str] = Field(description="The value of the override", default=None)
+    from_override: bool = Field(description="Is this protocol generated from an override", default=False)
 
     @property
     def file_path(self) -> Path:
         """Path of the file."""
+        if self.from_override:
+            return Path(
+                Path(__file__).resolve().parent.parent.parent,
+                os.getenv("FILES_FOLDER", "files"),
+                "protocols",
+                GENERATED_PROTOCOLS_FOLDER,
+                f"{self.file_stem}.{self.file_extension}",
+            )
         return Path(
             Path(__file__).resolve().parent.parent.parent,
             os.getenv("FILES_FOLDER", "files"),
             "protocols",
-            f"{self.file_extension}",
-            f"{self.file_name}.{self.file_extension}",
+            f"{self.file_stem}.{self.file_extension}",
         )
 
     @property
@@ -58,6 +70,6 @@ def labware_paths(self) -> list[Path]:
     def short_sha(self) -> str:
         """Short sha of the file."""
         # Hash the string using SHA-1
-        hash_object = hashlib.sha1(self.file_name.encode())
+        hash_object = hashlib.sha1(self.file_stem.encode())
         # Convert to hexadecimal and truncate
         return hash_object.hexdigest()[:10]

app-testing/automation/data/protocol_registry.py

@@ -0,0 +1,62 @@
+import os
+from typing import Optional
+
+from rich.console import Console
+from rich.panel import Panel
+
+from automation.data.protocol import Protocol
+from automation.data.protocol_with_overrides import ProtocolWithOverrides
+from automation.data.protocols import Protocols
+from automation.data.protocols_with_overrides import ProtocolsWithOverrides
+
+
+class ProtocolRegistry:
+    def __init__(self) -> None:
+        self.protocols: Protocols = Protocols()
+        self.protocols_with_overrides: ProtocolsWithOverrides = ProtocolsWithOverrides()
+        self.protocols_to_test: Optional[list[Protocol]] = self._what_protocols()
+
+    def _what_protocols(self) -> Optional[list[Protocol]]:
+        protocol_names: Optional[str] = os.environ.get("APP_ANALYSIS_TEST_PROTOCOLS")
+        override_protocol_names: Optional[str] = os.environ.get("APP_ANALYSIS_TEST_PROTOCOLS_WITH_OVERRIDES")
+        protocols_to_test: list[Protocol] = []
+        if protocol_names:
+            for protocol_name in [x.strip() for x in protocol_names.split(",")]:
+                protocol: Protocol = getattr(self.protocols, protocol_name)  # raises
+                protocols_to_test.append(protocol)
+        if override_protocol_names:
+            for protocol_with_overrides__name in [x.strip() for x in override_protocol_names.split(",")]:
+                protocol_with_overrides: ProtocolWithOverrides = getattr(
+                    self.protocols_with_overrides, protocol_with_overrides__name
+                )  # raises
+                if protocol_with_overrides.protocols is not None:
+                    protocols_to_test.extend(protocol_with_overrides.protocols)
+        if protocols_to_test == []:
+            return None
+        return protocols_to_test
+
+    def all_defined_protocols(self) -> list[Protocol]:
+        return [getattr(self.protocols, prop) for prop in dir(self.protocols) if "__" not in prop]
+
+    def all_defined_protocols_with_overrides(self) -> list[ProtocolWithOverrides]:
+        return [getattr(self.protocols_with_overrides, prop) for prop in dir(self.protocols_with_overrides) if "__" not in prop]
+
+
+def main() -> None:
+    console = Console()
+    protocol_registry = ProtocolRegistry()
+    console.print("protocols for APP_ANALYSIS_TEST_PROTOCOLS")
+    console.print(Panel('Formatted for .env APP_ANALYSIS_TEST_PROTOCOLS="'))
+    sorted_stems = sorted([p.file_stem for p in protocol_registry.all_defined_protocols()])
+    console.print('APP_ANALYSIS_TEST_PROTOCOLS="')
+    console.print(",\n".join(sorted_stems))
+    console.print('"')
+    console.print(Panel('Formatted for .env APP_ANALYSIS_TEST_PROTOCOLS_WITH_OVERRIDES="'))
+    console.print('APP_ANALYSIS_TEST_PROTOCOLS_WITH_OVERRIDES="')
+    sorted_stems = sorted([p.file_stem for p in protocol_registry.all_defined_protocols_with_overrides()])
+    console.print(",\n".join(sorted_stems))
+    console.print('"')
+
+
+if __name__ == "__main__":
+    main()