diff --git a/pyproject.toml b/pyproject.toml index da9f3381d..44a15b92b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ dependencies = [ "backports.entry_points_selectable", "defusedxml", # For safely parsing XML files - "pydantic<2", # Locked to <2 by cygwin terminal + "pydantic>=2", "requests", "rich", "werkzeug", @@ -47,7 +47,7 @@ client = [ "websocket-client", ] developer = [ - "bump-my-version<0.11.0", # Version control + "bump-my-version", # Version control "ipykernel", # Enable interactive coding with VS Code and Jupyter Notebook "pre-commit", # Formatting, linting, type checking, etc. "pytest", # Test code functionality @@ -61,7 +61,7 @@ server = [ "aiohttp", "cryptography", "fastapi[standard]", - "ispyb", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python; v10.0.0: sqlalchemy <2, mysql-connector-python >=8.0.32 + "ispyb>=10.2.4", # Responsible for setting requirements for SQLAlchemy and mysql-connector-python; "jinja2", "mrcfile", "numpy", @@ -74,7 +74,7 @@ server = [ "sqlmodel", "stomp-py<=8.1.0", # 8.1.1 (released 2024-04-06) doesn't work with our project "uvicorn[standard]", - "zocalo", + "zocalo>=1", ] [project.urls] Bug-Tracker = "https://github.com/DiamondLightSource/python-murfey/issues" diff --git a/src/murfey/client/instance_environment.py b/src/murfey/client/instance_environment.py index 1f8a642e9..ff44505f4 100644 --- a/src/murfey/client/instance_environment.py +++ b/src/murfey/client/instance_environment.py @@ -8,7 +8,7 @@ from typing import Dict, List, NamedTuple, Optional from urllib.parse import ParseResult -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from murfey.client.watchdir import DirWatcher @@ -53,9 +53,7 @@ class MurfeyInstanceEnvironment(BaseModel): murfey_session: Optional[int] = None samples: Dict[Path, SampleInfo] = {} - class Config: - validate_assignment: bool = True - arbitrary_types_allowed: bool = True + model_config = ConfigDict(arbitrary_types_allowed=True) def clear(self): self.sources = [] diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index e712ffccb..5942b22fd 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -198,7 +198,7 @@ def validate_form(form: dict, model: BaseModel) -> bool: try: convert = lambda x: None if x == "None" else x validated = model(**{k: convert(v) for k, v in form.items()}) - log.info(validated.dict()) + log.info(validated.model_dump()) return True except (AttributeError, ValidationError) as e: log.warning(f"Form validation failed: {str(e)}") diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index c95b18dfe..c2a4d8783 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -153,7 +153,7 @@ def start_multigrid_watcher( demo=True, do_transfer=True, processing_enabled=not watcher_spec.skip_existing_processing, - _machine_config=watcher_spec.configuration.dict(), + _machine_config=watcher_spec.configuration.model_dump(), token=tokens.get(session_id, "token"), data_collection_parameters=data_collection_parameters.get(label, {}), rsync_restarts=watcher_spec.rsync_restarts, @@ -167,7 +167,7 @@ def start_multigrid_watcher( (watcher_spec.source / d).mkdir(exist_ok=True) watchers[session_id] = MultigridDirWatcher( watcher_spec.source, - watcher_spec.configuration.dict(), + watcher_spec.configuration.model_dump(), skip_existing_processing=watcher_spec.skip_existing_processing, ) watchers[session_id].subscribe( @@ -237,7 +237,7 @@ def register_processing_parameters( session_id: MurfeySessionID, proc_param_block: ProcessingParameterBlock ): data_collection_parameters[proc_param_block.label] = {} - for k, v in proc_param_block.params.dict().items(): + for k, v in proc_param_block.params.model_dump().items(): data_collection_parameters[proc_param_block.label][k] = v return {"success": True} diff --git a/src/murfey/server/api/clem.py b/src/murfey/server/api/clem.py index 606e18496..feb5b67c0 100644 --- a/src/murfey/server/api/clem.py +++ b/src/murfey/server/api/clem.py @@ -10,7 +10,7 @@ from backports.entry_points_selectable import entry_points from fastapi import APIRouter -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from sqlalchemy.exc import NoResultFound from sqlmodel import Session, select @@ -707,10 +707,8 @@ class AlignAndMergeParams(BaseModel): flatten: Literal["mean", "min", "max", ""] = "" align_across: Literal["enabled", ""] = "" - @validator( - "images", - pre=True, - ) + @field_validator("images", mode="before") + @classmethod def parse_stringified_list(cls, value): if isinstance(value, str): try: diff --git a/src/murfey/server/api/spa.py b/src/murfey/server/api/spa.py index 5a2a2c737..4b1b1fc5c 100644 --- a/src/murfey/server/api/spa.py +++ b/src/murfey/server/api/spa.py @@ -20,12 +20,14 @@ def _cryolo_model_path(visit: str, instrument_name: str) -> Path: machine_config = get_machine_config(instrument_name=instrument_name)[ instrument_name ] - if machine_config.model_search_directory: + if machine_config.picking_model_search_directory: visit_directory = ( machine_config.rsync_basepath / str(datetime.now().year) / visit ) possible_models = list( - (visit_directory / machine_config.model_search_directory).glob("*.h5") + (visit_directory / machine_config.picking_model_search_directory).glob( + "*.h5" + ) ) if possible_models: return sorted(possible_models, key=lambda x: x.stat().st_ctime)[-1] diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py index 3c6b74886..9f45315fc 100644 --- a/src/murfey/server/demo_api.py +++ b/src/murfey/server/demo_api.py @@ -15,7 +15,8 @@ from fastapi.responses import FileResponse, HTMLResponse from ispyb.sqlalchemy import BLSession from PIL import Image -from pydantic import BaseModel, BaseSettings +from pydantic import BaseModel +from pydantic_settings import BaseSettings from sqlalchemy import func from sqlmodel import col, select from werkzeug.utils import secure_filename diff --git a/src/murfey/server/main.py b/src/murfey/server/main.py index 96533fd7d..c18434bee 100644 --- a/src/murfey/server/main.py +++ b/src/murfey/server/main.py @@ -8,7 +8,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles from prometheus_client import make_asgi_app -from pydantic import BaseSettings +from pydantic_settings import BaseSettings import murfey.server import murfey.server.api.auth diff --git a/src/murfey/util/config.py b/src/murfey/util/config.py index ca8c65ed3..390f871de 100644 --- a/src/murfey/util/config.py +++ b/src/murfey/util/config.py @@ -8,7 +8,8 @@ import yaml from backports.entry_points_selectable import entry_points -from pydantic import BaseModel, BaseSettings, Extra, validator +from pydantic import BaseModel, Extra, field_validator +from pydantic_settings import BaseSettings class MachineConfig(BaseModel, extra=Extra.allow): # type: ignore @@ -58,7 +59,7 @@ class MachineConfig(BaseModel, extra=Extra.allow): # type: ignore upstream_data_download_directory: Optional[Path] = None # Set by microscope config upstream_data_tiff_locations: List[str] = ["processed"] # Location of CLEM TIFFs - model_search_directory: str = "processing" + picking_model_search_directory: str = "processing" initial_model_search_directory: str = "processing/initial_model" failure_queue: str = "" @@ -97,7 +98,7 @@ class Security(BaseModel): graylog_host: str = "" graylog_port: Optional[int] = None - @validator("graylog_port") + @field_validator("graylog_port") def check_port_present_if_host_is( cls, v: Optional[int], values: dict, **kwargs ) -> Optional[int]: diff --git a/src/murfey/util/models.py b/src/murfey/util/models.py index 82e5ce708..cf7bc350e 100644 --- a/src/murfey/util/models.py +++ b/src/murfey/util/models.py @@ -97,7 +97,7 @@ class ConnectionFileParameters(BaseModel): class SessionInfo(BaseModel): - session_id: Optional[int] + session_id: Optional[int] = None session_name: str = "" rescale: bool = True @@ -160,7 +160,7 @@ class Sample(BaseModel): sample_group_id: int sample_id: int subsample_id: int - image_path: Optional[Path] + image_path: Optional[Path] = None class BLSampleImageParameters(BaseModel): @@ -194,13 +194,13 @@ class SPAProcessFile(BaseModel): tag: str path: str description: str - processing_job: Optional[int] - data_collection_id: Optional[int] + processing_job: Optional[int] = None + data_collection_id: Optional[int] = None image_number: int - autoproc_program_id: Optional[int] - foil_hole_id: Optional[int] - pixel_size: Optional[float] - dose_per_frame: Optional[float] + autoproc_program_id: Optional[int] = None + foil_hole_id: Optional[int] = None + pixel_size: Optional[float] = None + dose_per_frame: Optional[float] = None mc_binning: Optional[int] = 1 gain_ref: Optional[str] = None extract_downscale: bool = True @@ -211,7 +211,7 @@ class SPAProcessFile(BaseModel): class ProcessingParametersSPA(BaseModel): tag: str dose_per_frame: float - gain_ref: Optional[str] + gain_ref: Optional[str] = None experiment_type: str voltage: float image_size_x: int @@ -222,12 +222,12 @@ class ProcessingParametersSPA(BaseModel): acquisition_software: str use_cryolo: bool symmetry: str - mask_diameter: Optional[int] - boxsize: Optional[int] + mask_diameter: Optional[int] = None + boxsize: Optional[int] = None downscale: bool - small_boxsize: Optional[int] + small_boxsize: Optional[int] = None eer_fractionation: int - particle_diameter: Optional[float] + particle_diameter: Optional[float] = None magnification: Optional[int] = None total_exposed_dose: Optional[float] = None c2aperture: Optional[float] = None @@ -236,14 +236,14 @@ class ProcessingParametersSPA(BaseModel): phase_plate: bool = False class Base(BaseModel): - dose_per_frame: Optional[float] - gain_ref: Optional[str] + dose_per_frame: Optional[float] = None + gain_ref: Optional[str] = None use_cryolo: bool symmetry: str - mask_diameter: Optional[int] - boxsize: Optional[int] + mask_diameter: Optional[int] = None + boxsize: Optional[int] = None downscale: bool - small_boxsize: Optional[int] + small_boxsize: Optional[int] = None eer_fractionation: int @@ -350,7 +350,7 @@ class CompletedTiltSeries(BaseModel): class PreprocessingParametersTomo(BaseModel): dose_per_frame: float frame_count: int - gain_ref: Optional[str] + gain_ref: Optional[str] = None experiment_type: str voltage: float image_size_x: int @@ -361,12 +361,12 @@ class PreprocessingParametersTomo(BaseModel): file_extension: str tag: str tilt_series_tag: str - eer_fractionation_file: Optional[str] + eer_fractionation_file: Optional[str] = None eer_fractionation: int class Base(BaseModel): dose_per_frame: float - gain_ref: Optional[str] + gain_ref: Optional[str] = None manual_tilt_offset: float eer_fractionation: int diff --git a/src/murfey/util/rsync.py b/src/murfey/util/rsync.py index 84953b39a..bcbb4b059 100644 --- a/src/murfey/util/rsync.py +++ b/src/murfey/util/rsync.py @@ -108,6 +108,7 @@ def _single_rsync( else: cmd.append(str(self._finaldir / sub_struct) + "/") self._transferring = True + runner = subprocess.run( cmd, capture_output=True, diff --git a/src/murfey/workflows/clem/register_align_and_merge_results.py b/src/murfey/workflows/clem/register_align_and_merge_results.py index b02c56a30..8ae14dbcd 100644 --- a/src/murfey/workflows/clem/register_align_and_merge_results.py +++ b/src/murfey/workflows/clem/register_align_and_merge_results.py @@ -7,7 +7,7 @@ from pathlib import Path from typing import Optional -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from sqlmodel import Session from murfey.util.db import CLEMImageSeries @@ -24,10 +24,8 @@ class AlignAndMergeResult(BaseModel): align_across: Optional[str] = None composite_image: Path - @validator( - "image_stacks", - pre=True, - ) + @field_validator("image_stacks", mode="before") + @classmethod def parse_stringified_list(cls, value): if isinstance(value, str): try: diff --git a/src/murfey/workflows/clem/register_preprocessing_results.py b/src/murfey/workflows/clem/register_preprocessing_results.py index 396f1109e..7ab6c359d 100644 --- a/src/murfey/workflows/clem/register_preprocessing_results.py +++ b/src/murfey/workflows/clem/register_preprocessing_results.py @@ -13,7 +13,7 @@ from ast import literal_eval from pathlib import Path -from pydantic import BaseModel, validator +from pydantic import BaseModel, field_validator from sqlmodel import Session, select from murfey.server import _transport_object @@ -221,10 +221,8 @@ class TIFFPreprocessingResult(BaseModel): number_of_members: int parent_tiffs: list[Path] - @validator( - "parent_tiffs", - pre=True, - ) + @field_validator("parent_tiffs", mode="before") + @classmethod def parse_stringified_list(cls, value): if isinstance(value, str): try: diff --git a/tests/cli/test_decrypt_password.py b/tests/cli/test_decrypt_password.py index 65952e5e8..f7b9a192a 100644 --- a/tests/cli/test_decrypt_password.py +++ b/tests/cli/test_decrypt_password.py @@ -13,7 +13,7 @@ def test_decrypt_password(capsys, tmp_path): crypto_key = Fernet.generate_key() security_config.crypto_key = crypto_key.decode("ascii") with open(tmp_path / "config.yaml", "w") as cfg: - yaml.dump(security_config.dict(), cfg) + yaml.dump(security_config.model_dump(), cfg) os.environ["MURFEY_SECURITY_CONFIGURATION"] = str(tmp_path / "config.yaml") password = "abcd" f = Fernet(crypto_key) diff --git a/tests/cli/test_generate_password.py b/tests/cli/test_generate_password.py index fa48e9cf2..e7d4ead05 100644 --- a/tests/cli/test_generate_password.py +++ b/tests/cli/test_generate_password.py @@ -12,7 +12,7 @@ def test_generate_password(capsys, tmp_path): crypto_key = Fernet.generate_key() security_config.crypto_key = crypto_key.decode("ascii") with open(tmp_path / "config.yaml", "w") as cfg: - yaml.dump(security_config.dict(), cfg) + yaml.dump(security_config.model_dump(), cfg) os.environ["MURFEY_SECURITY_CONFIGURATION"] = str(tmp_path / "config.yaml") run() captured = capsys.readouterr()