refactor: Migrate Python projects from Pydantic v1 to v2

api/Pipfile

@@ -5,7 +5,8 @@ name = "pypi"
 
 [packages]
 jsonschema = "==4.17.3"
-pydantic = "==1.10.12"
+pydantic = "==2.6.4"
+pydantic-settings = "==2.2.1"
 anyio = "==3.7.1"
 opentrons-shared-data = { editable = true, path = "../shared-data/python" }
 opentrons = { editable = true, path = "." }

api/mypy.ini

@@ -39,3 +39,7 @@ disallow_incomplete_defs = False
 disallow_untyped_defs = False
 disallow_untyped_calls = False
 disallow_incomplete_defs = False
+
+# TODO: fixed in Pydantic 2.7, see https://github.com/pydantic/pydantic/pull/9008
+[mypy-tests.opentrons.protocol_engine.execution.test_command_executor.*]
+no_strict_optional=True

api/setup.py

@@ -59,9 +59,9 @@ def get_version():
     f"opentrons-shared-data=={VERSION}",
     "aionotify==0.2.0",
     "anyio>=3.6.1,<4.0.0",
-    "jsonschema>=3.0.1,<4.18.0",
+    "jsonschema>=4.0.0,<5",
     "numpy>=1.20.0,<2",
-    "pydantic>=1.10.9,<2.0.0",
+    "pydantic>=2.0.0,<3",
     "pyserial>=3.5",
     "typing-extensions>=4.0.0,<5",
     "click>=8.0.0,<9",

api/src/opentrons/calibration_storage/deck_configuration.py

@@ -23,9 +23,9 @@ def serialize_deck_configuration(
     cutout_fixture_placements: List[CutoutFixturePlacement], last_modified: datetime
 ) -> bytes:
     """Serialize a deck configuration for storing on the filesystem."""
-    data = _DeckConfigurationModel.construct(
+    data = _DeckConfigurationModel.model_construct(
         cutoutFixtures=[
-            _CutoutFixturePlacementModel.construct(
+            _CutoutFixturePlacementModel.model_construct(
                 cutoutId=e.cutout_id, cutoutFixtureId=e.cutout_fixture_id
             )
             for e in cutout_fixture_placements

api/src/opentrons/calibration_storage/file_operators.py

@@ -112,7 +112,7 @@ def save_to_file(
 
 def serialize_pydantic_model(data: pydantic.BaseModel) -> bytes:
     """Safely serialize data from a Pydantic model into a form suitable for storing on disk."""
-    return data.json(by_alias=True).encode("utf-8")
+    return data.model_dump_json(by_alias=True).encode("utf-8")
 
 
 _ModelT = typing.TypeVar("_ModelT", bound=pydantic.BaseModel)
@@ -133,7 +133,7 @@ def deserialize_pydantic_model(
     Returns `None` if the file is missing or corrupt.
     """
     try:
-        return model.parse_raw(serialized)
+        return model.model_validate_json(serialized)
     except json.JSONDecodeError:
         _log.warning("Data is not valid JSON.", exc_info=True)
         return None

api/src/opentrons/calibration_storage/ot2/models/v1.py

@@ -1,19 +1,24 @@
 import typing
 
 from typing_extensions import Literal
-from pydantic import BaseModel, Field, validator
+from pydantic import field_validator, BaseModel, Field, PlainSerializer
 from datetime import datetime
 
 from opentrons_shared_data.pipette.dev_types import LabwareUri
 
 from opentrons.types import Point
 from opentrons.calibration_storage import types
 
+DatetimeType = typing.Annotated[
+    datetime,
+    PlainSerializer(lambda x: x.isoformat(), when_used="json"),
+]
+
 
 class CalibrationStatus(BaseModel):
     markedBad: bool = False
     source: typing.Optional[types.SourceType] = None
-    markedAt: typing.Optional[datetime] = None
+    markedAt: typing.Optional[DatetimeType] = None
 
 
 # Schemas used to store the data types
@@ -22,7 +27,7 @@ class CalibrationStatus(BaseModel):
 # they are currently saved in on the OT-2 to avoid a large migration.
 class TipLengthModel(BaseModel):
     tipLength: float = Field(..., description="Tip length data found from calibration.")
-    lastModified: datetime = Field(
+    lastModified: DatetimeType = Field(
         ..., description="The last time this tip length was calibrated."
     )
     source: types.SourceType = Field(
@@ -40,22 +45,19 @@ class TipLengthModel(BaseModel):
         ..., description="The tiprack hash associated with the tip length data."
     )
 
-    @validator("tipLength")
+    @field_validator("tipLength")
+    @classmethod
     def ensure_tip_length_positive(cls, tipLength: float) -> float:
         if tipLength < 0.0:
             raise ValueError("Tip Length must be a positive number")
         return tipLength
 
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}
-
 
 class DeckCalibrationModel(BaseModel):
     attitude: types.AttitudeMatrix = Field(
         ..., description="Attitude matrix for deck found from calibration."
     )
-    last_modified: typing.Optional[datetime] = Field(
+    last_modified: typing.Optional[DatetimeType] = Field(
         default=None, description="The last time this deck was calibrated."
     )
     source: types.SourceType = Field(
@@ -72,18 +74,14 @@ class DeckCalibrationModel(BaseModel):
         description="The status of the calibration data.",
     )
 
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}
-
 
 class InstrumentOffsetModel(BaseModel):
     offset: Point = Field(..., description="Instrument offset found from calibration.")
     tiprack: str = Field(..., description="Tiprack used to calibrate this offset")
     uri: str = Field(
         ..., description="The URI of the labware used for instrument offset"
     )
-    last_modified: datetime = Field(
+    last_modified: DatetimeType = Field(
         ..., description="The last time this instrument was calibrated."
     )
     source: types.SourceType = Field(
@@ -94,10 +92,6 @@ class InstrumentOffsetModel(BaseModel):
         description="The status of the calibration data.",
     )
 
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}
-
 
 # TODO(lc 09-19-2022) We need to refactor the calibration endpoints
 # so that we only need to use one data model schema. This model is a
@@ -112,7 +106,7 @@ class PipetteOffsetCalibration(BaseModel):
     uri: str = Field(
         ..., description="The URI of the labware used for instrument offset"
     )
-    last_modified: datetime = Field(
+    last_modified: DatetimeType = Field(
         ..., description="The last time this instrument was calibrated."
     )
     source: types.SourceType = Field(
@@ -123,10 +117,6 @@ class PipetteOffsetCalibration(BaseModel):
         description="The status of the calibration data.",
     )
 
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}
-
 
 # TODO(lc 09-19-2022) We need to refactor the calibration endpoints
 # so that we only need to use one data model schema. This model is a
@@ -137,7 +127,7 @@ class TipLengthCalibration(BaseModel):
         ..., description="The tiprack hash associated with this tip length data."
     )
     tipLength: float = Field(..., description="Tip length data found from calibration.")
-    lastModified: datetime = Field(
+    lastModified: DatetimeType = Field(
         ..., description="The last time this tip length was calibrated."
     )
     source: types.SourceType = Field(
@@ -151,12 +141,9 @@ class TipLengthCalibration(BaseModel):
         ..., description="The tiprack URI associated with the tip length data."
     )
 
-    @validator("tipLength")
+    @field_validator("tipLength")
+    @classmethod
     def ensure_tip_length_positive(cls, tipLength: float) -> float:
         if tipLength < 0.0:
             raise ValueError("Tip Length must be a positive number")
         return tipLength
-
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}

api/src/opentrons/calibration_storage/ot2/tip_length.py

@@ -26,6 +26,7 @@
 def _convert_tip_length_model_to_dict(
     to_dict: typing.Dict[LabwareUri, v1.TipLengthModel]
 ) -> typing.Dict[LabwareUri, typing.Any]:
+    # TODO[pydantic]: supported in Pydantic V2
     # This is a workaround since pydantic doesn't have a nice way to
     # add encoders when converting to a dict.
     dict_of_tip_lengths = {}

api/src/opentrons/calibration_storage/ot3/models/v1.py

@@ -3,7 +3,7 @@
 from typing_extensions import Literal
 from opentrons.hardware_control.modules.types import ModuleType
 from opentrons.hardware_control.types import OT3Mount
-from pydantic import BaseModel, Field, validator
+from pydantic import field_validator, BaseModel, Field, PlainSerializer
 from datetime import datetime
 
 from opentrons_shared_data.pipette.dev_types import LabwareUri
@@ -12,15 +12,21 @@
 from opentrons.calibration_storage import types
 
 
+DatetimeType = typing.Annotated[
+    datetime,
+    PlainSerializer(lambda x: x.isoformat(), when_used="json"),
+]
+
+
 class CalibrationStatus(BaseModel):
     markedBad: bool = False
     source: typing.Optional[types.SourceType] = None
-    markedAt: typing.Optional[datetime] = None
+    markedAt: typing.Optional[DatetimeType] = None
 
 
 class TipLengthModel(BaseModel):
     tipLength: float = Field(..., description="Tip length data found from calibration.")
-    lastModified: datetime = Field(
+    lastModified: DatetimeType = Field(
         ..., description="The last time this tip length was calibrated."
     )
     uri: typing.Union[LabwareUri, Literal[""]] = Field(
@@ -34,22 +40,19 @@ class TipLengthModel(BaseModel):
         description="The status of the calibration data.",
     )
 
-    @validator("tipLength")
+    @field_validator("tipLength")
+    @classmethod
     def ensure_tip_length_positive(cls, tipLength: float) -> float:
         if tipLength < 0.0:
             raise ValueError("Tip Length must be a positive number")
         return tipLength
 
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}
-
 
 class BeltCalibrationModel(BaseModel):
     attitude: types.AttitudeMatrix = Field(
         ..., description="Attitude matrix for belts found from calibration."
     )
-    lastModified: datetime = Field(
+    lastModified: DatetimeType = Field(
         ..., description="The last time this deck was calibrated."
     )
     source: types.SourceType = Field(
@@ -63,14 +66,10 @@ class BeltCalibrationModel(BaseModel):
         description="The status of the calibration data.",
     )
 
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}
-
 
 class InstrumentOffsetModel(BaseModel):
     offset: Point = Field(..., description="Instrument offset found from calibration.")
-    lastModified: datetime = Field(
+    lastModified: DatetimeType = Field(
         ..., description="The last time this instrument was calibrated."
     )
     source: types.SourceType = Field(
@@ -81,10 +80,6 @@ class InstrumentOffsetModel(BaseModel):
         description="The status of the calibration data.",
     )
 
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}
-
 
 class ModuleOffsetModel(BaseModel):
     offset: Point = Field(..., description="Module offset found from calibration.")
@@ -99,7 +94,7 @@ class ModuleOffsetModel(BaseModel):
         ...,
         description="The unique id of the instrument used to calibrate this module.",
     )
-    lastModified: datetime = Field(
+    lastModified: DatetimeType = Field(
         ..., description="The last time this module was calibrated."
     )
     source: types.SourceType = Field(
@@ -109,7 +104,3 @@ class ModuleOffsetModel(BaseModel):
         default_factory=CalibrationStatus,
         description="The status of the calibration data.",
     )
-
-    class Config:
-        json_encoders = {datetime: lambda obj: obj.isoformat()}
-        json_decoders = {datetime: lambda obj: datetime.fromisoformat(obj)}

api/src/opentrons/cli/analyze.py

@@ -81,20 +81,19 @@ async def _analyze(
         robot_type=protocol_source.robot_type, protocol_config=protocol_source.config
     )
     analysis = await runner.run(deck_configuration=[], protocol_source=protocol_source)
-
     if json_output:
-        results = AnalyzeResults.construct(
+        results = AnalyzeResults.model_construct(
             createdAt=datetime.now(tz=timezone.utc),
             files=[
-                ProtocolFile.construct(name=f.path.name, role=f.role)
+                ProtocolFile.model_construct(name=f.path.name, role=f.role)
                 for f in protocol_source.files
             ],
             config=(
-                JsonConfig.construct(
+                JsonConfig.model_construct(
                     schemaVersion=protocol_source.config.schema_version
                 )
                 if isinstance(protocol_source.config, JsonProtocolConfig)
-                else PythonConfig.construct(
+                else PythonConfig.model_construct(
                     apiVersion=protocol_source.config.api_version
                 )
             ),
@@ -110,7 +109,7 @@ async def _analyze(
         )
 
         await json_output.write_text(
-            results.json(exclude_none=True),
+            results.model_dump_json(exclude_none=True),
             encoding="utf-8",
         )
 

api/src/opentrons/execute.py

@@ -23,7 +23,7 @@
     Union,
 )
 
-from opentrons_shared_data.labware.labware_definition import LabwareDefinition
+from opentrons_shared_data.labware.models import LabwareDefinition
 from opentrons_shared_data.robot.dev_types import RobotType
 
 from opentrons import protocol_api, __version__, should_use_ot3
@@ -550,7 +550,9 @@ def _create_live_context_pe(
     # Non-async would use call_soon_threadsafe(), which makes the waiting harder.
     async def add_all_extra_labware() -> None:
         for labware_definition_dict in extra_labware.values():
-            labware_definition = LabwareDefinition.parse_obj(labware_definition_dict)
+            labware_definition = LabwareDefinition.model_validate(
+                labware_definition_dict
+            )
             pe.add_labware_definition(labware_definition)
 
     # Add extra_labware to ProtocolEngine, being careful not to modify ProtocolEngine from this

api/src/opentrons/hardware_control/emulation/settings.py

@@ -1,7 +1,8 @@
 from typing import List
 from opentrons.hardware_control.emulation.types import ModuleType
 from opentrons.hardware_control.emulation.util import TEMPERATURE_ROOM
-from pydantic import BaseSettings, BaseModel
+from pydantic import BaseModel
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class PipetteSettings(BaseModel):
@@ -113,8 +114,6 @@ class Settings(BaseSettings):
         emulator_port=9003, driver_port=9998
     )
     magdeck_proxy: ProxySettings = ProxySettings(emulator_port=9004, driver_port=9999)
-
-    class Config:
-        env_prefix = "OT_EMULATOR_"
+    model_config = SettingsConfigDict(env_prefix="OT_EMULATOR_")
 
     module_server: ModuleServerSettings = ModuleServerSettings()

api/src/opentrons/hardware_control/instruments/ot2/instrument_calibration.py

@@ -9,7 +9,7 @@
 from opentrons.types import Mount, Point
 from opentrons.hardware_control.types import OT3Mount
 
-from opentrons_shared_data.labware.labware_definition import LabwareDefinition
+from opentrons_shared_data.labware.models import LabwareDefinition
 
 if typing.TYPE_CHECKING:
     from opentrons_shared_data.pipette.dev_types import LabwareUri