feat(hardware-testing): update the post processing for multi sensor r…

…eadings (#15014)

<!--
Thanks for taking the time to open a pull request! Please make sure
you've read the "Opening Pull Requests" section of our Contributing
Guide:


https://github.com/Opentrons/opentrons/blob/edge/CONTRIBUTING.md#opening-pull-requests

To ensure your code is reviewed quickly and thoroughly, please fill out
the sections below to the best of your ability!
-->

# Overview
The 8 channel and 96 channel now collect data from both pressure sensors
during the liquid probe script, this PR allows the post processing
routine to handle that data and changes the format of final_report.csv
to include all of the data

<!--
Use this section to describe your pull-request at a high level. If the
PR addresses any open issues, please tag the issues here.
-->

# Test Plan

<!--
Use this section to describe the steps that you took to test your Pull
Request.
If you did not perform any testing provide justification why.

OT-3 Developers: You should default to testing on actual physical
hardware.
Once again, if you did not perform testing against hardware, justify
why.

Note: It can be helpful to write a test plan before doing development

Example Test Plan (HTTP API Change)

- Verified that new optional argument `dance-party` causes the robot to
flash its lights, move the pipettes,
then home.
- Verified that when you omit the `dance-party` option the robot homes
normally
- Added protocol that uses `dance-party` argument to G-Code Testing
Suite
- Ran protocol that did not use `dance-party` argument and everything
was successful
- Added unit tests to validate that changes to pydantic model are
correct

-->

# Changelog

<!--
List out the changes to the code in this PR. Please try your best to
categorize your changes and describe what has changed and why.

Example changelog:
- Fixed app crash when trying to calibrate an illegal pipette
- Added state to API to track pipette usage
- Updated API docs to mention only two pipettes are supported

IMPORTANT: MAKE SURE ANY BREAKING CHANGES ARE PROPERLY COMMUNICATED
-->

# Review requests

<!--
Describe any requests for your reviewers here.
-->

# Risk assessment

<!--
Carefully go over your pull request and look at the other parts of the
codebase it may affect. Look for the possibility, even if you think it's
small, that your change may affect some other part of the system - for
instance, changing return tip behavior in protocol may also change the
behavior of labware calibration.

Identify the other parts of the system your codebase may affect, so that
in addition to your own review and testing, other people who may not
have the system internalized as much as you can focus their attention
and testing there.
-->

hardware-testing/hardware_testing/liquid_sense/__main__.py

@@ -80,7 +80,6 @@ class RunArgs:
     pipette: InstrumentContext
     pipette_tag: str
     git_description: str
-    robot_serial: str
     recorder: GravimetricRecorder
     pipette_volume: int
     pipette_channels: int
@@ -140,7 +139,6 @@ def _get_protocol_context(cls, args: argparse.Namespace) -> ProtocolContext:
     def build_run_args(cls, args: argparse.Namespace) -> "RunArgs":
         """Build."""
         _ctx = RunArgs._get_protocol_context(args)
-        robot_serial = helpers._get_robot_serial(_ctx.is_simulating())
         run_id, start_time = create_run_id_and_start_time()
         environment_sensor = asair_sensor.BuildAsairSensor(
             _ctx.is_simulating() or args.ignore_env
@@ -161,7 +159,10 @@ def build_run_args(cls, args: argparse.Namespace) -> "RunArgs":
         pipette_tag = helpers._get_tag_from_pipette(pipette, False, False)
 
         if args.trials == 0:
-            trials = 10
+            if args.channels < 96:
+                trials = 10
+            else:
+                trials = 7
         else:
             trials = args.trials
 
@@ -195,7 +196,6 @@ def build_run_args(cls, args: argparse.Namespace) -> "RunArgs":
         # go ahead and store the meta data now
         store_serial_numbers(
             report,
-            robot_serial,
             pipette_tag,
             scale.read_serial_number(),
             environment_sensor.get_serial(),
@@ -220,7 +220,6 @@ def build_run_args(cls, args: argparse.Namespace) -> "RunArgs":
             pipette=pipette,
             pipette_tag=pipette_tag,
             git_description=git_description,
-            robot_serial=robot_serial,
             recorder=recorder,
             pipette_volume=args.pipette,
             pipette_channels=args.channels,

hardware-testing/hardware_testing/liquid_sense/execute.py

@@ -176,43 +176,53 @@ def run(tip: int, run_args: RunArgs) -> None:
         lpc_offset = run_args.dial_indicator.read_stable()
         run_args.pipette._retract()
 
-    def _get_baseline() -> float:
+    def _get_tip_offset() -> float:
+        tip_offset = 0.0
+        if run_args.dial_indicator is not None:
+            run_args.pipette.move_to(dial_well.top())
+            tip_offset = run_args.dial_indicator.read_stable()
+            run_args.pipette._retract()
+        return tip_offset
+
+    def _get_target_height() -> float:
         run_args.pipette.pick_up_tip(tips[0])
         del tips[: run_args.pipette_channels]
         liquid_height = _jog_to_find_liquid_height(
             run_args.ctx, run_args.pipette, test_well
         )
         target_height = test_well.bottom(liquid_height).point.z
-
         run_args.pipette._retract()
-        tip_offset = 0.0
-        if run_args.dial_indicator is not None:
-            run_args.pipette.move_to(dial_well.top())
-            tip_offset = run_args.dial_indicator.read_stable()
-            run_args.pipette._retract()
-        if run_args.return_tip:
-            run_args.pipette.return_tip()
-        else:
-            run_args.pipette.drop_tip()
+        return target_height
 
-        env_data = run_args.environment_sensor.get_reading()
+    target_height = _get_target_height()
+    tip_offset = _get_tip_offset()
 
-        store_baseline_trial(
-            run_args.test_report,
-            tip,
-            target_height,
-            env_data.relative_humidity,
-            env_data.temperature,
-            test_well.top().point.z - target_height,
-            tip_offset - lpc_offset,
-        )
-        return target_height
+    if run_args.return_tip:
+        run_args.pipette.return_tip()
+    else:
+        run_args.pipette.drop_tip()
+
+    env_data = run_args.environment_sensor.get_reading()
+
+    store_baseline_trial(
+        run_args.test_report,
+        tip,
+        target_height,
+        env_data.relative_humidity,
+        env_data.temperature,
+        test_well.top().point.z - target_height,
+        tip_offset - lpc_offset,
+    )
 
     trials_before_jog = run_args.trials_before_jog
-    tip_offset = 0.0
+
     for trial in range(run_args.trials):
-        if trial % trials_before_jog == 0:
-            tip_offset = _get_baseline()
+        if trial > 0 and trial % trials_before_jog == 0:
+            target_height = _get_target_height()
+            if run_args.return_tip:
+                run_args.pipette.return_tip()
+            else:
+                run_args.pipette.drop_tip()
 
         ui.print_info(f"Picking up {tip}ul tip")
         run_args.pipette.pick_up_tip(tips[0])
@@ -225,7 +235,6 @@ def _get_baseline() -> float:
         run_args.pipette.blow_out()
         tip_length_offset = 0.0
         if run_args.dial_indicator is not None:
-
             run_args.pipette._retract()
             run_args.pipette.move_to(dial_well.top())
             tip_length_offset = tip_offset - run_args.dial_indicator.read_stable()
@@ -257,6 +266,7 @@ def _get_baseline() -> float:
             start_pos[Axis.Z_L] - end_pos[Axis.Z_L],
             plunger_start - end_pos[Axis.P_L],
             tip_length_offset,
+            target_height,
         )
         ui.print_info(
             f"\n\n Z axis start pos {start_pos[Axis.Z_L]} end pos {end_pos[Axis.Z_L]}"

hardware-testing/hardware_testing/liquid_sense/post_process.py

@@ -21,6 +21,25 @@
 }
 
 
+def _get_pressure_results(result_file: str) -> Tuple[float, float, float, List[float]]:
+    z_velocity: float = 0.0
+    p_velocity: float = 0.0
+    threshold: float = 0.0
+    pressures: List[float] = []
+    with open(result_file, newline="") as trial_csv:
+        trial_reader = csv.reader(trial_csv)
+        i = 0
+        for row in trial_reader:
+            if i == 1:
+                z_velocity = float(row[2])
+                p_velocity = float(row[3])
+                threshold = float(row[4])
+            if i > 1:
+                pressures.append(float(row[1]))
+            i += 1
+    return z_velocity, p_velocity, threshold, pressures
+
+
 def process_csv_directory(  # noqa: C901
     data_directory: str, tips: List[int], trials: int, make_graph: bool = False
 ) -> None:
@@ -29,15 +48,22 @@ def process_csv_directory(  # noqa: C901
     summary: str = [f for f in csv_files if "CSVReport" in f][0]
     final_report_file: str = f"{data_directory}/final_report.csv"
     # initialize our data structs
-    pressure_csvs = [f for f in csv_files if "pressure_sensor_data" in f]
-    pressure_results_files: Dict[int, List[str]] = {}
-    pressure_results: Dict[int, Dict[int, List[float]]] = {}
+    primary_pressure_csvs = [f for f in csv_files if "PRIMARY" in f]
+    secondary_pressure_csvs = [f for f in csv_files if "SECONDARY" in f]
+    primary_pressure_results_files: Dict[int, List[str]] = {}
+    secondary_pressure_results_files: Dict[int, List[str]] = {}
+    pressure_results: Dict[int, Dict[int, List[Tuple[float, float]]]] = {}
     results_settings: Dict[int, Dict[int, Tuple[float, float, float]]] = {}
     tip_offsets: Dict[int, List[float]] = {}
     p_offsets: Dict[int, List[float]] = {}
     meniscus_travel: float = 0
     for tip in tips:
-        pressure_results_files[tip] = [f for f in pressure_csvs if f"tip{tip}" in f]
+        primary_pressure_results_files[tip] = [
+            f for f in primary_pressure_csvs if f"tip{tip}" in f
+        ]
+        secondary_pressure_results_files[tip] = [
+            f for f in secondary_pressure_csvs if f"tip{tip}" in f
+        ]
         pressure_results[tip] = {}
         results_settings[tip] = {}
         tip_offsets[tip] = []
@@ -50,22 +76,35 @@ def process_csv_directory(  # noqa: C901
     # read in all of the pressure csvs into one big struct so we can process them
     for tip in tips:
         for trial in range(trials):
-            with open(
-                f"{data_directory}/{pressure_results_files[tip][trial]}", newline=""
-            ) as trial_csv:
-                trial_reader = csv.reader(trial_csv)
-                i = 0
-                for row in trial_reader:
-                    if i == 1:
-                        results_settings[tip][trial] = (
-                            float(row[2]),
-                            float(row[3]),
-                            float(row[4]),
-                        )
-                    if i > 1:
-                        pressure_results[tip][trial].append(float(row[1]))
-                    i += 1
-                max_results_len = max([i - 2, max_results_len])
+            z_velocity: float = 0.0
+            p_velocity: float = 0.0
+            threshold: float = 0.0
+            primary_pressures: List[float] = []
+            secondary_pressures: List[float] = []
+            if trial < len(primary_pressure_results_files[tip]):
+                (
+                    z_velocity,
+                    p_velocity,
+                    threshold,
+                    primary_pressures,
+                ) = _get_pressure_results(
+                    f"{data_directory}/{primary_pressure_results_files[tip][trial]}"
+                )
+            if trial < len(secondary_pressure_results_files[tip]):
+                (
+                    z_velocity,
+                    p_velocity,
+                    threshold,
+                    secondary_pressures,
+                ) = _get_pressure_results(
+                    f"{data_directory}/{secondary_pressure_results_files[tip][trial]}"
+                )
+            results_settings[tip][trial] = (z_velocity, p_velocity, threshold)
+            for i in range(max(len(primary_pressures), len(secondary_pressures))):
+                p = primary_pressures[i] if i < len(primary_pressures) else 0.0
+                s = secondary_pressures[i] if i < len(secondary_pressures) else 0.0
+                pressure_results[tip][trial].append((p, s))
+            max_results_len = max([len(pressure_results[tip][trial]), max_results_len])
     # start writing the final report csv
     with open(f"{data_directory}/{summary}", newline="") as summary_csv:
         summary_reader = csv.reader(summary_csv)
@@ -76,11 +115,11 @@ def process_csv_directory(  # noqa: C901
             for row in summary_reader:
                 final_report_writer.writerow(row)
                 s += 1
-                if s == 45:
+                if s == 44:
                     meniscus_travel = float(row[6])
-                if s >= 46 and s < 46 + (trials * len(tips)):
+                if s >= 45 and s < 45 + (trials * len(tips)):
                     # while processing this grab the tip offsets from the summary
-                    tip_offsets[tips[int((s - 46) / trials)]].append(float(row[8]))
+                    tip_offsets[tips[int((s - 45) / trials)]].append(float(row[8]))
             # summary_reader.line_num is the last line in the summary that has text
             pressures_start_line = summary_reader.line_num + 3
             # calculate where the start and end of each block of data we want to graph
@@ -102,7 +141,12 @@ def process_csv_directory(  # noqa: C901
             pressure_header_row = ["time", ""]
             for i in range(trials):
                 pressure_header_row.extend(
-                    [f"pressure T{i+1}", f"z_travel T{i+1}", f"p_travel T{i+1}"]
+                    [
+                        f"primary pressure T{i+1}",
+                        f"secondary pressure T{i+1}",
+                        f"z_travel T{i+1}",
+                        f"p_travel T{i+1}",
+                    ]
                 )
 
             # we want to line up the z height's of each trial at time==0
@@ -153,9 +197,11 @@ def process_csv_directory(  # noqa: C901
                         pressure_row.append("")
                     for trial in range(trials):
                         if i < len(pressure_results[tip][trial]):
-                            pressure_row.append(f"{pressure_results[tip][trial][i]}")
+                            pressure_row.append(f"{pressure_results[tip][trial][i][0]}")
+                            pressure_row.append(f"{pressure_results[tip][trial][i][1]}")
                         else:
                             pressure_row.append("")
+                            pressure_row.append("")
                         pressure_row.append(
                             f"{results_settings[tip][trial][0] * time - tip_offsets[tip][trial]}"
                         )

hardware-testing/hardware_testing/liquid_sense/report.py

@@ -41,7 +41,6 @@ def build_serial_number_section() -> CSVSection:
     return CSVSection(
         title="SERIAL-NUMBERS",
         lines=[
-            CSVLine("robot", [str]),
             CSVLine("git_description", [str]),
             CSVLine("pipette", [str]),
             CSVLine("scale", [str]),
@@ -71,7 +70,7 @@ def build_config_section() -> CSVSection:
 def build_trials_section(trials: int, tips: List[int]) -> CSVSection:
     """Build section."""
     lines: List[Union[CSVLine, CSVLineRepeating]] = [
-        CSVLine("trial_number", [str, str, str, str, str, str, str, str])
+        CSVLine("trial_number", [str, str, str, str, str, str, str, str, str])
     ]
     lines.extend(
         [
@@ -86,7 +85,7 @@ def build_trials_section(trials: int, tips: List[int]) -> CSVSection:
         [
             CSVLine(
                 f"trial-{t + 1}-{tip}ul",
-                [float, float, float, float, float, float, float, float],
+                [float, float, float, float, float, float, float, float, float],
             )
             for tip in tips
             for t in range(trials)
@@ -116,14 +115,12 @@ def build_results_section(tips: List[int]) -> CSVSection:
 
 def store_serial_numbers(
     report: CSVReport,
-    robot: str,
     pipette: str,
     scale: str,
     environment: str,
     git_description: str,
 ) -> None:
     """Report serial numbers."""
-    report("SERIAL-NUMBERS", "robot", [robot])
     report("SERIAL-NUMBERS", "git_description", [git_description])
     report("SERIAL-NUMBERS", "pipette", [pipette])
     report("SERIAL-NUMBERS", "scale", [scale])
@@ -195,6 +192,7 @@ def store_trial(
     z_travel: float,
     plunger_travel: float,
     tip_length_offset: float,
+    target_height: float,
 ) -> None:
     """Report Trial."""
     report(
@@ -209,6 +207,7 @@ def store_trial(
             plunger_travel,
             tip_length_offset,
             height + tip_length_offset,
+            target_height,
         ],
     )
 
@@ -258,6 +257,7 @@ def build_ls_report(
             "plunger_travel",
             "tip_length_offset",
             "adjusted_height",
+            "target_height",
         ],
     )
     return report

hardware-testing/hardware_testing/protocols/liquid_sense_lpc/liquid_sense_ot3_p1000_96.py

@@ -8,9 +8,9 @@
 SLOT_DIAL = 5
 SLOTS_TIPRACK = {
     # TODO: add slot 12 when tipracks are disposable
-    50: [2, 3, 6, 7, 8, 9, 10, 11],
-    200: [2, 3, 6, 7, 8, 9, 10, 11],  # NOTE: ignored during calibration
-    1000: [2, 3, 6, 7, 8, 9, 10, 11],  # NOTE: ignored during calibration
+    50: [1, 2, 3, 6, 7, 8, 9, 10, 11],
+    200: [1, 2, 3, 6, 7, 8, 9, 10, 11],  # NOTE: ignored during calibration
+    1000: [1, 2, 3, 6, 7, 8, 9, 10, 11],  # NOTE: ignored during calibration
 }
 
 LABWARE_ON_SCALE = "nest_1_reservoir_195ml"

hardware/opentrons_hardware/hardware_control/tool_sensors.py

@@ -234,7 +234,7 @@ async def _setup_pressure_sensors(
 
     for sensor in sensors:
         pressure_sensor = PressureSensor.build(
-            sensor_id=sensor_id,
+            sensor_id=sensor,
             node_id=tool,
             stop_threshold=threshold_fixed_point,
         )