From 8e08a28e1a054ee98493d7a3b8269dacb2cb3a37 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Thu, 26 Sep 2024 13:44:20 +0200 Subject: [PATCH 01/18] add relion5 meta data input entry and parsing of the meta data --- src/pytom_tm/entry_points.py | 45 ++++++++++++---- src/pytom_tm/io.py | 99 ++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 11 deletions(-) diff --git a/src/pytom_tm/entry_points.py b/src/pytom_tm/entry_points.py index a493f60..d6a356d 100644 --- a/src/pytom_tm/entry_points.py +++ b/src/pytom_tm/entry_points.py @@ -19,6 +19,7 @@ ParseDefocus, BetweenZeroAndOne, ParseGPUIndices, + parse_relion5_star_data, ) from pytom_tm.tmjob import load_json_to_tmjob from os import urandom @@ -731,7 +732,7 @@ def match_template(argv=None): "--tilt-angles", nargs="+", type=str, - required=True, + required=False, action=ParseTiltAngles, help="Tilt angles of the tilt-series, either the minimum and maximum values of " "the tilts (e.g. --tilt-angles -59.1 60.1) or a .rawtlt/.tlt file with all the " @@ -897,6 +898,13 @@ def match_template(argv=None): help="Specify a seed for the random number generator used for phase " "randomization for consistent results!", ) + additional_group.add_argument( + "--relion5-tomograms-star", + type=pathlib.Path, + action=CheckFileExists, + required=False, + help="TO-DO", + ) device_group = parser.add_argument_group("Device control") device_group.add_argument( "-g", @@ -922,6 +930,11 @@ def match_template(argv=None): args = parser.parse_args(argv) logging.basicConfig(level=args.log, force=True) + # parse CTF phase correction + phase_flip_correction = False + if args.tomogram_ctf_model is not None and args.tomogram_ctf_model == "phase-flip": + phase_flip_correction = True + # combine ctf values to ctf_params list of dicts ctf_params = None if args.defocus is not None: @@ -935,12 +948,6 @@ def match_template(argv=None): "the required parameters (amplitude-contrast, " "spherical-abberation or voltage) is/are missing." ) - phase_flip_correction = False - if ( - args.tomogram_ctf_model is not None - and args.tomogram_ctf_model == "phase-flip" - ): - phase_flip_correction = True ctf_params = [ { "defocus": defocus * 1e-6, @@ -953,6 +960,22 @@ def match_template(argv=None): for defocus in args.defocus ] + if args.relion5_tomograms_star is not None: + voxel_size, tilt_angles, dose_accumulation, ctf_params = ( + parse_relion5_star_data( + args.relion5_tomograms_star, + args.tomograms, + phase_flip_correction=phase_flip_correction, + phase_shift=args.phase_shift, + ) + ) + per_tilt_weighting = True + else: + voxel_size = args.voxel_size + tilt_angles = args.tilt_angles + dose_accumulation = args.dose_accumulation + per_tilt_weighting = args.per_tilt_weighting + if args.angular_search is None and args.particle_diameter is None: raise ValueError( "Either the angular search should be specifically set or a particle " @@ -970,16 +993,16 @@ def match_template(argv=None): mask_is_spherical=True if args.non_spherical_mask is None else (not args.non_spherical_mask), - tilt_angles=args.tilt_angles, - tilt_weighting=args.per_tilt_weighting, + tilt_angles=tilt_angles, + tilt_weighting=per_tilt_weighting, search_x=args.search_x, search_y=args.search_y, search_z=args.search_z, tomogram_mask=args.tomogram_mask, - voxel_size=args.voxel_size_angstrom, + voxel_size=voxel_size, low_pass=args.low_pass, high_pass=args.high_pass, - dose_accumulation=args.dose_accumulation, + dose_accumulation=dose_accumulation, ctf_data=ctf_params, whiten_spectrum=args.spectral_whitening, rotational_symmetry=args.z_axis_rotational_symmetry, diff --git a/src/pytom_tm/io.py b/src/pytom_tm/io.py index 0a1e8aa..144e77c 100644 --- a/src/pytom_tm/io.py +++ b/src/pytom_tm/io.py @@ -8,6 +8,9 @@ from operator import attrgetter from typing import Optional, Union +# new imports +import starfile + class ParseLogging(argparse.Action): """argparse.Action subclass to parse logging parameter from input scripts. Users can @@ -490,3 +493,99 @@ def read_defocus_file(file_name: pathlib.Path) -> list[float, ...]: return read_txt_file(file_name) else: raise ValueError("Defocus file needs to have format .defocus or .txt") + + +def parse_relion5_star_data( + tomograms_star_path: pathlib.Path, + tomogram_path: pathlib.Path, + phase_flip_correction: bool = False, + phase_shift: float = 0.0, +) -> tuple[float, list[float, ...], list[float, ...], dict]: + """Read RELION5 metadata from a project directory. + + Parameters + ---------- + tomograms_star_path: pathlib.Path + the tomograms.star from a RELION5 reconstruct job contains invariable metadata + and points to a tilt series star file with fitted values + tomogram_path: pathlib.Path + path to the tomogram for template matching; we use the name to pattern match in + the RELION5 star file + phase_flip_correction: bool, default False + phase_shift: float, default 0.0 + + Returns + ------- + tomogram_voxel_size, tilt_angles, dose_accumulation, ctf_params: + tuple[float, list[float, ...], list[float, ...], list[dict, ...]] + """ + tomogram_id = tomogram_path.stem + tomograms_star_data = starfile.read(tomograms_star_path) + + # match the tomo_id and check if viable + matches = [ + (i, x) + for i, x in enumerate(tomograms_star_data["rlnTomoName"]) + if x in tomogram_id + ] + if len(matches) > 1: + raise ValueError( + "Multiple matches of tomogram id in RELION5 STAR file. " "Aborting..." + ) + elif len(matches) == 0: + raise ValueError("No match of tomogram id in RELION5 STAR file. Aborting...") + else: + tomogram_meta_data = tomograms_star_data.loc[matches[0][0]] + + # grab the path to tilt series star where tilt angles, defocus and dose are + # annotated + tilt_series_star_path = pathlib.Path( + tomogram_meta_data["rlnTomoTiltSeriesStarFile"] + ) + # update the path to a location we can actually find from CD + tilt_series_star_path = tomograms_star_path.parent.joinpath("tilt_series").joinpath( + tilt_series_star_path.name + ) + tilt_series_star_data = starfile.read(tilt_series_star_path) + + # we extract tilt angles, dose accumulation and ctf params + # TODO We need to have an internal structure for tilt series meta data + tilt_angles = list(tilt_series_star_data["rlnTomoNominalStageTiltAngle"]) + dose_accumulation = list(tilt_series_star_data["rlnMicrographPreExposure"]) + + # _rlnTomoName # 1 + # _rlnVoltage # 2 + # _rlnSphericalAberration # 3 + # _rlnAmplitudeContrast # 4 + # _rlnMicrographOriginalPixelSize # 5 + # _rlnTomoHand # 6 + # _rlnTomoTiltSeriesPixelSize # 7 + # _rlnTomoTiltSeriesStarFile # 8 + # _rlnTomoTomogramBinning # 9 + # _rlnTomoSizeX # 10 + # _rlnTomoSizeY # 11 + # _rlnTomoSizeZ # 12 + # _rlnTomoReconstructedTomogram # 13 + tomogram_voxel_size = ( + tomogram_meta_data["rlnTomoTiltSeriesPixelSize"] + * tomogram_meta_data["rlnTomoTomogramBinning"] + ) + + ctf_params = [ + { + "defocus": defocus * 1e-6, + "amplitude_contrast": tomogram_meta_data["rlnAmplitudeContrast"], + "voltage": tomogram_meta_data["rlnVoltage"] * 1e3, + "spherical_aberration": tomogram_meta_data["rlnSphericalAberration"] * 1e-3, + "flip_phase": phase_flip_correction, + "phase_shift_deg": phase_shift, # Unclear to me where the phase + # shifts are annotated in RELION5 tilt series file + } + for defocus in ( + tilt_series_star_data.rlnDefocusV + tilt_series_star_data.rlnDefocusU + ) + / 2 + * 1e-4 + ] + + return tomogram_voxel_size, tilt_angles, dose_accumulation, ctf_params From 60fc428f1c93e45d3355cf91e6fcba7895c78e89 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Thu, 26 Sep 2024 14:57:17 +0200 Subject: [PATCH 02/18] fix arg name type --- src/pytom_tm/entry_points.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytom_tm/entry_points.py b/src/pytom_tm/entry_points.py index d6a356d..062c66a 100644 --- a/src/pytom_tm/entry_points.py +++ b/src/pytom_tm/entry_points.py @@ -964,7 +964,7 @@ def match_template(argv=None): voxel_size, tilt_angles, dose_accumulation, ctf_params = ( parse_relion5_star_data( args.relion5_tomograms_star, - args.tomograms, + args.tomogram, phase_flip_correction=phase_flip_correction, phase_shift=args.phase_shift, ) From dc652d9ba17745574c2358f2f5c91b73a18bd34e Mon Sep 17 00:00:00 2001 From: McHaillet Date: Thu, 26 Sep 2024 15:25:30 +0200 Subject: [PATCH 03/18] fix argument name --- src/pytom_tm/entry_points.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytom_tm/entry_points.py b/src/pytom_tm/entry_points.py index 062c66a..153f853 100644 --- a/src/pytom_tm/entry_points.py +++ b/src/pytom_tm/entry_points.py @@ -971,7 +971,7 @@ def match_template(argv=None): ) per_tilt_weighting = True else: - voxel_size = args.voxel_size + voxel_size = args.voxel_size_angstrom tilt_angles = args.tilt_angles dose_accumulation = args.dose_accumulation per_tilt_weighting = args.per_tilt_weighting From 3e2a35911e51cfce776f9045cf8b18ae191ce396 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Fri, 27 Sep 2024 11:10:23 +0200 Subject: [PATCH 04/18] add switch to remove rec_ --- src/pytom_tm/extract.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pytom_tm/extract.py b/src/pytom_tm/extract.py index c8ef50c..b7c6eac 100644 --- a/src/pytom_tm/extract.py +++ b/src/pytom_tm/extract.py @@ -277,6 +277,11 @@ def extract_particles( pixel_size = job.voxel_size tomogram_id = job.tomo_id + # remove relion5 reconstructed tomogram name as it messes with linking the tilt + # series id when extracting subtomos + if relion5_compat and tomogram_id.startswith("rec_"): + tomogram_id = tomogram_id[4:] + data = [] scores = [] From a1c47996575857423daae44449603e5184c35a3a Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 17:59:06 +0200 Subject: [PATCH 05/18] remove: temporary comment --- src/pytom_tm/io.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pytom_tm/io.py b/src/pytom_tm/io.py index 144e77c..99aa5b1 100644 --- a/src/pytom_tm/io.py +++ b/src/pytom_tm/io.py @@ -4,13 +4,11 @@ import logging import numpy.typing as npt import numpy as np +import starfile from contextlib import contextmanager from operator import attrgetter from typing import Optional, Union -# new imports -import starfile - class ParseLogging(argparse.Action): """argparse.Action subclass to parse logging parameter from input scripts. Users can From 4feb0f6c20af3bc4b2aaf6350794e010e23a501d Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 17:59:38 +0200 Subject: [PATCH 06/18] add: test data for relion5 tilt series metadata --- .../job009/tilt_series/tomo200528_107.star | 67 +++++++++++++++++++ .../Tomograms/job009/tomograms.star | 30 +++++++++ 2 files changed, 97 insertions(+) create mode 100644 tests/Data/relion5_project_example/Tomograms/job009/tilt_series/tomo200528_107.star create mode 100644 tests/Data/relion5_project_example/Tomograms/job009/tomograms.star diff --git a/tests/Data/relion5_project_example/Tomograms/job009/tilt_series/tomo200528_107.star b/tests/Data/relion5_project_example/Tomograms/job009/tilt_series/tomo200528_107.star new file mode 100644 index 0000000..e754c31 --- /dev/null +++ b/tests/Data/relion5_project_example/Tomograms/job009/tilt_series/tomo200528_107.star @@ -0,0 +1,67 @@ + +# version 50001 + +data_tomo200528_107 + +loop_ +_rlnMicrographMovieName #1 +_rlnTomoTiltMovieFrameCount #2 +_rlnTomoNominalStageTiltAngle #3 +_rlnTomoNominalTiltAxisAngle #4 +_rlnMicrographPreExposure #5 +_rlnTomoNominalDefocus #6 +_rlnMicrographName #7 +_rlnMicrographMetadata #8 +_rlnAccumMotionTotal #9 +_rlnAccumMotionEarly #10 +_rlnAccumMotionLate #11 +_rlnCtfImage #12 +_rlnDefocusU #13 +_rlnDefocusV #14 +_rlnCtfAstigmatism #15 +_rlnDefocusAngle #16 +_rlnCtfFigureOfMerit #17 +_rlnCtfMaxResolution #18 +_rlnCtfIceRingDensity #19 +_rlnTomoXTilt #20 +_rlnTomoYTilt #21 +_rlnTomoZRot #22 +_rlnTomoXShiftAngst #23 +_rlnTomoYShiftAngst #24 +_rlnCtfScalefactor #25 +../raw/tomo200528_107_-51.0_May31_00.24.27.tif 8 -50.99360 -91.00000 68.200000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-51_0_May31_00_24_27.mrc MotionCorr/job002/__/raw/tomo200528_107_-51_0_May31_00_24_27.star 2.507579 0.000000 2.507579 CtfFind/job003/__/raw/tomo200528_107_-51_0_May31_00_24_27.ctf:mrc 24013.296875 22401.580078 1611.716797 -58.53130 0.025491 25.000000 1.671510 0.000000 -50.99000 -90.54510 9.537168 -19.87772 0.629456 +../raw/tomo200528_107_-48.0_May31_00.24.05.tif 8 -47.99250 -91.00000 66.000000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-48_0_May31_00_24_05.mrc MotionCorr/job002/__/raw/tomo200528_107_-48_0_May31_00_24_05.star 1.974866 0.000000 1.974866 CtfFind/job003/__/raw/tomo200528_107_-48_0_May31_00_24_05.ctf:mrc 21833.007812 20866.742188 966.265625 -37.32801 0.021794 11.173266 4.233130 0.000000 -47.99000 -90.54510 -1.81365 -4.21863 0.669260 +../raw/tomo200528_107_-45.0_May31_00.23.53.tif 8 -44.99330 -91.00000 63.800000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-45_0_May31_00_23_53.mrc MotionCorr/job002/__/raw/tomo200528_107_-45_0_May31_00_23_53.star 2.057416 0.000000 2.057416 CtfFind/job003/__/raw/tomo200528_107_-45_0_May31_00_23_53.ctf:mrc 33115.417969 31932.328125 1183.089844 -14.02393 0.021466 9.194667 3.024260 0.000000 -44.99000 -90.54510 13.319624 -160.25614 0.707230 +../raw/tomo200528_107_-42.0_May31_00.21.42.tif 8 -41.99320 -91.00000 52.800000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-42_0_May31_00_21_42.mrc MotionCorr/job002/__/raw/tomo200528_107_-42_0_May31_00_21_42.star 2.147246 0.000000 2.147246 CtfFind/job003/__/raw/tomo200528_107_-42_0_May31_00_21_42.ctf:mrc 32927.902344 32296.234375 631.667969 -54.50343 0.020782 12.259555 0.667460 0.000000 -41.99000 -90.54510 -14.45746 31.116476 0.743262 +../raw/tomo200528_107_-39.0_May31_00.21.23.tif 8 -38.99250 -91.00000 50.600000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-39_0_May31_00_21_23.mrc MotionCorr/job002/__/raw/tomo200528_107_-39_0_May31_00_21_23.star 2.514207 0.000000 2.514207 CtfFind/job003/__/raw/tomo200528_107_-39_0_May31_00_21_23.ctf:mrc 25491.113281 25207.378906 283.734375 -2.17321 0.020923 25.000000 1.527350 0.000000 -38.99000 -90.54510 -11.72665 35.140292 0.777256 +../raw/tomo200528_107_-36.0_May31_00.20.57.tif 8 -35.99140 -91.00000 48.400000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-36_0_May31_00_20_57.mrc MotionCorr/job002/__/raw/tomo200528_107_-36_0_May31_00_20_57.star 2.101238 0.000000 2.101238 CtfFind/job003/__/raw/tomo200528_107_-36_0_May31_00_20_57.ctf:mrc 30034.322266 29555.056641 479.265625 -23.16141 0.024842 15.485754 1.682500 0.000000 -35.99000 -90.54510 41.719076 40.282984 0.809120 +../raw/tomo200528_107_-33.0_May31_00.20.34.tif 8 -32.99270 -91.00000 46.200000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-33_0_May31_00_20_34.mrc MotionCorr/job002/__/raw/tomo200528_107_-33_0_May31_00_20_34.star 1.623584 0.000000 1.623584 CtfFind/job003/__/raw/tomo200528_107_-33_0_May31_00_20_34.ctf:mrc 31693.958984 30160.537109 1533.421875 79.133553 0.020255 10.897383 4.368910 0.000000 -32.99000 -90.54510 -65.52234 -171.07424 0.838766 +../raw/tomo200528_107_-30.0_May31_00.19.31.tif 8 -29.99260 -91.00000 39.600000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-30_0_May31_00_19_31.mrc MotionCorr/job002/__/raw/tomo200528_107_-30_0_May31_00_19_31.star 1.881861 0.000000 1.881861 CtfFind/job003/__/raw/tomo200528_107_-30_0_May31_00_19_31.ctf:mrc 32093.007812 31556.515625 536.492188 -15.63826 0.020251 11.463480 0.568960 0.000000 -29.99000 -90.54510 37.450452 -146.17968 0.866113 +../raw/tomo200528_107_-27.0_May31_00.19.08.tif 8 -26.99140 -91.00000 37.400000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-27_0_May31_00_19_08.mrc MotionCorr/job002/__/raw/tomo200528_107_-27_0_May31_00_19_08.star 1.918317 0.000000 1.918317 CtfFind/job003/__/raw/tomo200528_107_-27_0_May31_00_19_08.ctf:mrc 31673.710938 31001.462891 672.248047 -24.74508 0.023158 7.952144 3.658200 0.000000 -26.99000 -90.54510 -7.60629 -62.92428 0.891086 +../raw/tomo200528_107_-24.0_May31_00.17.57.tif 8 -23.99230 -91.00000 30.800000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-24_0_May31_00_17_57.mrc MotionCorr/job002/__/raw/tomo200528_107_-24_0_May31_00_17_57.star 1.697119 0.000000 1.697119 CtfFind/job003/__/raw/tomo200528_107_-24_0_May31_00_17_57.ctf:mrc 31358.437500 29207.830078 2150.607422 64.128052 0.026186 10.634795 6.089830 0.000000 -23.99000 -90.54510 -30.90270 79.445368 0.913616 +../raw/tomo200528_107_-21.0_May31_00.17.45.tif 8 -20.99210 -91.00000 28.600000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-21_0_May31_00_17_45.mrc MotionCorr/job002/__/raw/tomo200528_107_-21_0_May31_00_17_45.star 1.919050 0.000000 1.919050 CtfFind/job003/__/raw/tomo200528_107_-21_0_May31_00_17_45.ctf:mrc 32090.488281 30359.730469 1730.757812 -18.93306 0.018624 12.980706 2.238780 0.000000 -20.99000 -90.54510 24.761812 41.555296 0.933643 +../raw/tomo200528_107_-18.0_May31_00.16.28.tif 8 -17.99100 -91.00000 22.000000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-18_0_May31_00_16_28.mrc MotionCorr/job002/__/raw/tomo200528_107_-18_0_May31_00_16_28.star 2.138721 0.000000 2.138721 CtfFind/job003/__/raw/tomo200528_107_-18_0_May31_00_16_28.ctf:mrc 30980.386719 29664.351562 1316.035156 -28.24077 0.024562 9.491269 6.506850 0.000000 -17.99000 -90.54510 -28.33739 16.055612 0.951110 +../raw/tomo200528_107_-15.0_May31_00.16.15.tif 8 -14.99130 -91.00000 19.800000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-15_0_May31_00_16_15.mrc MotionCorr/job002/__/raw/tomo200528_107_-15_0_May31_00_16_15.star 1.867401 0.000000 1.867401 CtfFind/job003/__/raw/tomo200528_107_-15_0_May31_00_16_15.ctf:mrc 30517.896484 30187.144531 330.751953 57.878517 0.027967 7.675548 1.382470 0.000000 -14.99000 -90.54510 5.201308 102.488352 0.965971 +../raw/tomo200528_107_-12.0_May31_00.14.59.tif 8 -11.99120 -91.00000 13.200000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-12_0_May31_00_14_59.mrc MotionCorr/job002/__/raw/tomo200528_107_-12_0_May31_00_14_59.star 1.924177 0.000000 1.924177 CtfFind/job003/__/raw/tomo200528_107_-12_0_May31_00_14_59.ctf:mrc 30597.599609 30174.507812 423.091797 -25.53507 0.045169 6.216113 3.004180 0.000000 -11.99000 -90.54510 -17.49860 37.083240 0.978184 +../raw/tomo200528_107_-9.0_May31_00.14.40.tif 8 -8.99102 -91.00000 11.000000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-9_0_May31_00_14_40.mrc MotionCorr/job002/__/raw/tomo200528_107_-9_0_May31_00_14_40.star 2.043660 0.000000 2.043660 CtfFind/job003/__/raw/tomo200528_107_-9_0_May31_00_14_40.ctf:mrc 30758.089844 30192.785156 565.304688 30.080853 0.036622 7.742877 4.107880 0.000000 -8.99000 -90.54510 -18.04511 47.187604 0.987716 +../raw/tomo200528_107_-6.0_May31_00.13.16.tif 8 -5.99087 -91.00000 4.400000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-6_0_May31_00_13_16.mrc MotionCorr/job002/__/raw/tomo200528_107_-6_0_May31_00_13_16.star 1.720566 0.000000 1.720566 CtfFind/job003/__/raw/tomo200528_107_-6_0_May31_00_13_16.ctf:mrc 31068.300781 30334.824219 733.476562 28.493963 0.035382 6.789908 6.105910 0.000000 -5.99000 -90.54510 12.157648 34.307600 0.994540 +../raw/tomo200528_107_-3.0_May31_00.12.50.tif 8 -2.99071 -91.00000 2.200000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_-3_0_May31_00_12_50.mrc MotionCorr/job002/__/raw/tomo200528_107_-3_0_May31_00_12_50.star 1.710414 0.000000 1.710414 CtfFind/job003/__/raw/tomo200528_107_-3_0_May31_00_12_50.ctf:mrc 31434.527344 31207.625000 226.902344 2.733182 0.042002 6.896000 4.368160 0.000000 -2.99000 -90.54510 12.357632 47.375520 0.998639 +../raw/tomo200528_107_0.0_May31_00.12.25.tif 8 0.008460 -91.00000 0.000000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_0_0_May31_00_12_25.mrc MotionCorr/job002/__/raw/tomo200528_107_0_0_May31_00_12_25.star 1.689892 0.832943 0.856950 CtfFind/job003/__/raw/tomo200528_107_0_0_May31_00_12_25.ctf:mrc 33063.156250 32802.414062 260.742188 -48.22010 0.040529 5.807158 4.426570 0.000000 0.010000 -90.54510 -9.36304 3.739356 1.000000 +../raw/tomo200528_107_3.0_May31_00.13.45.tif 8 3.006660 -91.00000 6.600000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_3_0_May31_00_13_45.mrc MotionCorr/job002/__/raw/tomo200528_107_3_0_May31_00_13_45.star 1.687404 0.000000 1.687404 CtfFind/job003/__/raw/tomo200528_107_3_0_May31_00_13_45.ctf:mrc 31362.593750 31193.765625 168.828125 -41.80921 0.045307 6.442978 3.090510 0.000000 3.010000 -90.54510 0.000000 0.000000 0.998620 +../raw/tomo200528_107_6.0_May31_00.14.12.tif 8 6.009250 -91.00000 8.800000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_6_0_May31_00_14_12.mrc MotionCorr/job002/__/raw/tomo200528_107_6_0_May31_00_14_12.star 1.519432 0.000000 1.519432 CtfFind/job003/__/raw/tomo200528_107_6_0_May31_00_14_12.ctf:mrc 31011.164062 30629.984375 381.179688 -4.44485 0.032017 5.769203 4.353410 0.000000 6.010000 -90.54510 -0.97061 -32.70945 0.994504 +../raw/tomo200528_107_9.0_May31_00.15.29.tif 8 9.006960 -91.00000 15.400000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_9_0_May31_00_15_29.mrc MotionCorr/job002/__/raw/tomo200528_107_9_0_May31_00_15_29.star 1.387599 0.000000 1.387599 CtfFind/job003/__/raw/tomo200528_107_9_0_May31_00_15_29.ctf:mrc 31166.060547 30905.615234 260.445312 -39.72199 0.045062 7.294942 5.645830 0.000000 9.010000 -90.54510 -15.95045 -42.48798 0.987661 +../raw/tomo200528_107_12.0_May31_00.15.50.tif 8 12.008600 -91.00000 17.600000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_12_0_May31_00_15_50.mrc MotionCorr/job002/__/raw/tomo200528_107_12_0_May31_00_15_50.star 1.339977 0.000000 1.339977 CtfFind/job003/__/raw/tomo200528_107_12_0_May31_00_15_50.ctf:mrc 30863.488281 30270.964844 592.523438 -24.98944 0.039216 7.235147 3.925220 0.000000 12.010000 -90.54510 -15.08328 -42.38799 0.978111 +../raw/tomo200528_107_15.0_May31_00.16.50.tif 8 15.006300 -91.00000 24.200000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_15_0_May31_00_16_50.mrc MotionCorr/job002/__/raw/tomo200528_107_15_0_May31_00_16_50.star 1.948747 0.000000 1.948747 CtfFind/job003/__/raw/tomo200528_107_15_0_May31_00_16_50.ctf:mrc 30469.888672 30228.898438 240.990234 31.616282 0.028301 9.099876 5.606470 0.000000 15.010000 -90.54510 -12.50072 -52.87336 0.965881 +../raw/tomo200528_107_18.0_May31_00.17.19.tif 8 18.006000 -91.00000 26.400000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_18_0_May31_00_17_19.mrc MotionCorr/job002/__/raw/tomo200528_107_18_0_May31_00_17_19.star 1.878197 0.000000 1.878197 CtfFind/job003/__/raw/tomo200528_107_18_0_May31_00_17_19.ctf:mrc 31611.832031 30522.597656 1089.234375 63.395622 0.035621 7.675548 2.390130 0.000000 18.010000 -90.54510 -7.80972 -57.32472 0.951003 +../raw/tomo200528_107_21.0_May31_00.18.23.tif 8 21.004200 -91.00000 33.000000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_21_0_May31_00_18_23.mrc MotionCorr/job002/__/raw/tomo200528_107_21_0_May31_00_18_23.star 1.788701 0.000000 1.788701 CtfFind/job003/__/raw/tomo200528_107_21_0_May31_00_18_23.ctf:mrc 30805.947266 30222.228516 583.718750 -10.79854 0.020743 10.030545 1.363630 0.000000 21.000000 -90.54510 -4.70652 -76.75076 0.933580 +../raw/tomo200528_107_24.0_May31_00.18.51.tif 8 24.006300 -91.00000 35.200000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_24_0_May31_00_18_51.mrc MotionCorr/job002/__/raw/tomo200528_107_24_0_May31_00_18_51.star 1.573359 0.000000 1.573359 CtfFind/job003/__/raw/tomo200528_107_24_0_May31_00_18_51.ctf:mrc 25957.093750 25680.773438 276.320312 38.434002 0.016900 9.390298 1.013530 0.000000 24.010000 -90.54510 -17.35206 -66.76535 0.913474 +../raw/tomo200528_107_27.0_May31_00.19.56.tif 8 27.003500 -91.00000 41.800000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_27_0_May31_00_19_56.mrc MotionCorr/job002/__/raw/tomo200528_107_27_0_May31_00_19_56.star 1.622335 0.000000 1.622335 CtfFind/job003/__/raw/tomo200528_107_27_0_May31_00_19_56.ctf:mrc 26932.492188 26743.253906 189.238281 88.215157 0.023010 8.406552 2.891090 0.000000 27.000000 -90.54510 -8.77688 -84.11051 0.891007 +../raw/tomo200528_107_30.0_May31_00.20.18.tif 8 30.004100 -91.00000 44.000000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_30_0_May31_00_20_18.mrc MotionCorr/job002/__/raw/tomo200528_107_30_0_May31_00_20_18.star 2.250957 0.000000 2.250957 CtfFind/job003/__/raw/tomo200528_107_30_0_May31_00_20_18.ctf:mrc 35397.195312 34839.859375 557.335938 -62.36766 0.026163 5.254095 2.171230 0.000000 30.000000 -90.54510 -9.82508 -88.51878 0.866025 +../raw/tomo200528_107_33.0_May31_00.22.15.tif 8 33.002300 -91.00000 55.000000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_33_0_May31_00_22_15.mrc MotionCorr/job002/__/raw/tomo200528_107_33_0_May31_00_22_15.star 2.071795 0.000000 2.071795 CtfFind/job003/__/raw/tomo200528_107_33_0_May31_00_22_15.ctf:mrc 30971.498047 30903.699219 67.798828 -15.16932 0.026457 10.508190 1.621130 0.000000 33.000000 -90.54510 -6.12020 -108.94818 0.838671 +../raw/tomo200528_107_36.0_May31_00.22.36.tif 8 36.003400 -91.00000 57.200000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_36_0_May31_00_22_36.mrc MotionCorr/job002/__/raw/tomo200528_107_36_0_May31_00_22_36.star 2.559800 0.000000 2.559800 CtfFind/job003/__/raw/tomo200528_107_36_0_May31_00_22_36.ctf:mrc 29352.312500 28813.498047 538.814453 -66.51581 0.024554 14.711466 1.477750 0.000000 36.000000 -90.54510 -11.82492 -104.89333 0.809017 +../raw/tomo200528_107_39.0_May31_00.23.04.tif 8 39.002600 -91.00000 59.400000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_39_0_May31_00_23_04.mrc MotionCorr/job002/__/raw/tomo200528_107_39_0_May31_00_23_04.star 2.437858 0.000000 2.437858 CtfFind/job003/__/raw/tomo200528_107_39_0_May31_00_23_04.ctf:mrc 31977.171875 30433.412109 1543.759766 -85.98605 0.026984 12.432225 2.089010 0.000000 39.000000 -90.54510 -9.27857 -115.17182 0.777146 +../raw/tomo200528_107_42.0_May31_00.23.25.tif 8 42.001800 -91.00000 61.600000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_42_0_May31_00_23_25.mrc MotionCorr/job002/__/raw/tomo200528_107_42_0_May31_00_23_25.star 2.203855 0.000000 2.203855 CtfFind/job003/__/raw/tomo200528_107_42_0_May31_00_23_25.ctf:mrc 29368.339844 28876.535156 491.804688 36.837719 0.018514 12.980706 0.803030 0.000000 42.000000 -90.54510 -14.29541 -126.59849 0.743145 +../raw/tomo200528_107_45.0_May31_00.24.57.tif 8 45.001500 -91.00000 70.400000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_45_0_May31_00_24_57.mrc MotionCorr/job002/__/raw/tomo200528_107_45_0_May31_00_24_57.star 2.445997 0.000000 2.445997 CtfFind/job003/__/raw/tomo200528_107_45_0_May31_00_24_57.ctf:mrc 27901.367188 27063.689453 837.677734 -8.03546 0.034693 10.263814 0.812400 0.000000 45.000000 -90.54510 -4.13070 -139.49229 0.707107 +../raw/tomo200528_107_48.0_May31_00.25.25.tif 8 48.001100 -91.00000 72.600000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_48_0_May31_00_25_25.mrc MotionCorr/job002/__/raw/tomo200528_107_48_0_May31_00_25_25.star 2.526043 0.000000 2.526043 CtfFind/job003/__/raw/tomo200528_107_48_0_May31_00_25_25.ctf:mrc 38656.203125 38004.054688 652.148438 21.104927 0.025635 7.742877 1.022170 0.000000 48.000000 -90.54510 -18.65885 -126.15542 0.669131 +../raw/tomo200528_107_51.0_May31_00.25.46.tif 8 51.000800 -91.00000 74.800000 -3.00000 MotionCorr/job002/__/raw/tomo200528_107_51_0_May31_00_25_46.mrc MotionCorr/job002/__/raw/tomo200528_107_51_0_May31_00_25_46.star 3.072073 0.000000 3.072073 CtfFind/job003/__/raw/tomo200528_107_51_0_May31_00_25_46.ctf:mrc 31305.253906 29558.539062 1746.714844 -47.77417 0.039377 11.033600 1.047970 0.000000 51.000000 -90.54510 -14.10404 -143.64368 0.629320 + diff --git a/tests/Data/relion5_project_example/Tomograms/job009/tomograms.star b/tests/Data/relion5_project_example/Tomograms/job009/tomograms.star new file mode 100644 index 0000000..170ccea --- /dev/null +++ b/tests/Data/relion5_project_example/Tomograms/job009/tomograms.star @@ -0,0 +1,30 @@ + +# version 50001 + +data_global + +loop_ +_rlnTomoName #1 +_rlnVoltage #2 +_rlnSphericalAberration #3 +_rlnAmplitudeContrast #4 +_rlnMicrographOriginalPixelSize #5 +_rlnTomoHand #6 +_rlnTomoTiltSeriesPixelSize #7 +_rlnTomoTiltSeriesStarFile #8 +_rlnTomoTomogramBinning #9 +_rlnTomoSizeX #10 +_rlnTomoSizeY #11 +_rlnTomoSizeZ #12 +_rlnTomoReconstructedTomogram #13 +tomo200528_100 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_100.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_100.mrc +tomo200528_101 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_101.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_101.mrc +tomo200528_102 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_102.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_102.mrc +tomo200528_103 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_103.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_103.mrc +tomo200528_104 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_104.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_104.mrc +tomo200528_107 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_107.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_107.mrc +tomo200528_108 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_108.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_108.mrc +tomo200528_110 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_110.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_110.mrc +tomo200528_111 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_111.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_111.mrc +tomo200528_112 200.000000 2.700000 0.080000 1.724000 -1.00000 1.724000 Tomograms/job009/tilt_series/tomo200528_112.star 8.120650 4000 4000 2000 Tomograms/job009/tomograms/rec_tomo200528_112.mrc + From a12a191346a1e3f775abc97e4d205240dabe5de8 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 18:21:13 +0200 Subject: [PATCH 07/18] add: entry point test for relion5 metadata input --- src/pytom_tm/entry_points.py | 5 +++++ tests/test_entry_points.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/pytom_tm/entry_points.py b/src/pytom_tm/entry_points.py index 153f853..f95f96a 100644 --- a/src/pytom_tm/entry_points.py +++ b/src/pytom_tm/entry_points.py @@ -971,6 +971,11 @@ def match_template(argv=None): ) per_tilt_weighting = True else: + if args.tilt_angles is None: + raise ValueError( + "Without tilt angles the missing wedge cannot be calculated. A " + "minimal run requires tilt angles." + ) voxel_size = args.voxel_size_angstrom tilt_angles = args.tilt_angles dose_accumulation = args.dose_accumulation diff --git a/tests/test_entry_points.py b/tests/test_entry_points.py index 27557c3..4c6bd50 100644 --- a/tests/test_entry_points.py +++ b/tests/test_entry_points.py @@ -37,6 +37,10 @@ DEFOCUS_IMOD = ( pathlib.Path(__file__).parent.joinpath("Data").joinpath("test_imod.defocus") ) +RELION5_TOMOGRAMS_STAR = pathlib.Path(__file__).parent.joinpath( + "Data/relion5_project_example/Tomograms/job009/tomograms.star" +) +RELION5_TOMOGRAM = TEST_DATA.joinpath("rec_tomo200528_107.mrc") # Initial logging level LOG_LEVEL = logging.getLogger().level @@ -58,6 +62,7 @@ def setUpClass(cls) -> None: io.write_mrc(TEMPLATE, np.zeros((5, 5, 5), dtype=np.float32), 1) io.write_mrc(MASK, np.zeros((5, 5, 5), dtype=np.float32), 1) io.write_mrc(TOMOGRAM, np.zeros((10, 10, 10), dtype=np.float32), 1) + io.write_mrc(RELION5_TOMOGRAM, np.zeros((10, 10, 10), dtype=np.float32), 1) np.savetxt(TILT_ANGLES, np.linspace(-50, 50, 35)) np.savetxt(DOSE, np.linspace(0, 100, 35)) np.savetxt(DEFOCUS, np.ones(35) * 3000) @@ -67,6 +72,7 @@ def tearDownClass(cls) -> None: TEMPLATE.unlink() MASK.unlink() TOMOGRAM.unlink() + RELION5_TOMOGRAM.unlink() TILT_ANGLES.unlink() DOSE.unlink() DEFOCUS.unlink() @@ -197,3 +203,27 @@ def start(arg_dict): # simplify run start(arguments) self.assertIn("gpu indices", dump.getvalue()) dump.close() + + # test relion5 metadata reading + arguments = defaults.copy() + [ + arguments.pop(x) + for x in [ + "--tilt-angles", + "--per-tilt-weighting", + "--dose-accumulation", + "--defocus", + "--amplitude-contrast", + "--spherical-aberration", + "--voltage", + ] + ] + arguments["-v"] = str(RELION5_TOMOGRAM) + arguments["--relion5-tomograms-star"] = str(RELION5_TOMOGRAMS_STAR) + start(arguments) + + with self.assertRaises( + ValueError, msg="Missing tilt angles should raise an error." + ): + arguments.pop("--relion5-tomograms-star") + start(arguments) From 418416ebc483c4fde6745ae9615ec4cc41334376 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 18:33:50 +0200 Subject: [PATCH 08/18] add: test for io function --- src/pytom_tm/io.py | 10 ++++------ tests/test_io.py | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/pytom_tm/io.py b/src/pytom_tm/io.py index 99aa5b1..09c332e 100644 --- a/src/pytom_tm/io.py +++ b/src/pytom_tm/io.py @@ -526,14 +526,12 @@ def parse_relion5_star_data( for i, x in enumerate(tomograms_star_data["rlnTomoName"]) if x in tomogram_id ] - if len(matches) > 1: + if len(matches) == 1: + tomogram_meta_data = tomograms_star_data.loc[matches[0][0]] + else: raise ValueError( - "Multiple matches of tomogram id in RELION5 STAR file. " "Aborting..." + "Multiple or zero matches of tomogram id in RELION5 STAR file. Aborting..." ) - elif len(matches) == 0: - raise ValueError("No match of tomogram id in RELION5 STAR file. Aborting...") - else: - tomogram_meta_data = tomograms_star_data.loc[matches[0][0]] # grab the path to tilt series star where tilt angles, defocus and dose are # annotated diff --git a/tests/test_io.py b/tests/test_io.py index 6163bbe..ee7a0da 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -6,7 +6,7 @@ import numpy as np import mrcfile -from pytom_tm.io import read_mrc, read_mrc_meta_data, write_mrc +from pytom_tm.io import read_mrc, read_mrc_meta_data, write_mrc, parse_relion5_star_data FAILING_MRC = pathlib.Path(__file__).parent.joinpath( pathlib.Path("Data/human_ribo_mask_32_8_5.mrc") @@ -15,6 +15,9 @@ CORRUPT_MRC = pathlib.Path(__file__).parent.joinpath( pathlib.Path("Data/header_only.mrc") ) +RELION5_TOMOGRAMS_STAR = pathlib.Path(__file__).parent.joinpath( + "Data/relion5_project_example/Tomograms/job009/tomograms.star" +) class TestBrokenMRC(unittest.TestCase): @@ -85,3 +88,20 @@ def test_cast_warning(self): write_mrc(fname, array, 1.0) self.assertEqual(len(cm.output), 1) self.assertIn("np.float32", cm.output[0]) + + def test_parse_relion5_star_data(self): + tomogram = pathlib.Path("rec_tomo200528_107.mrc") + meta_data = parse_relion5_star_data(RELION5_TOMOGRAMS_STAR, tomogram) + self.assertEqual(len(meta_data), 4) + self.assertIsInstance(meta_data[0], float) + self.assertIsInstance(meta_data[1], list) + self.assertIsInstance(meta_data[2], list) + self.assertIsInstance(meta_data[3], list) + self.assertIsInstance(meta_data[3][0], dict) + + # test name mismatch + tomogram = pathlib.Path("tomogram.mrc") + with self.assertRaises( + ValueError, msg="Unmatching tomograms name should " "raise an error." + ): + parse_relion5_star_data(RELION5_TOMOGRAMS_STAR, tomogram) From bd21cace9efa1c869549192d228134bd2c6da6b9 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 18:34:15 +0200 Subject: [PATCH 09/18] fix:string --- tests/test_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_io.py b/tests/test_io.py index ee7a0da..cc31a23 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -102,6 +102,6 @@ def test_parse_relion5_star_data(self): # test name mismatch tomogram = pathlib.Path("tomogram.mrc") with self.assertRaises( - ValueError, msg="Unmatching tomograms name should " "raise an error." + ValueError, msg="Unmatching tomograms name should raise an error." ): parse_relion5_star_data(RELION5_TOMOGRAMS_STAR, tomogram) From ff2bf93766565c393ff01758ea563854f229b9eb Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 18:41:19 +0200 Subject: [PATCH 10/18] make sure defocus handedness is also parsed and update tests accordingly --- src/pytom_tm/entry_points.py | 5 +++-- src/pytom_tm/io.py | 15 +++++++++++---- tests/test_io.py | 3 ++- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/pytom_tm/entry_points.py b/src/pytom_tm/entry_points.py index f95f96a..68c64bc 100644 --- a/src/pytom_tm/entry_points.py +++ b/src/pytom_tm/entry_points.py @@ -961,7 +961,7 @@ def match_template(argv=None): ] if args.relion5_tomograms_star is not None: - voxel_size, tilt_angles, dose_accumulation, ctf_params = ( + voxel_size, tilt_angles, dose_accumulation, ctf_params, defocus_handedness = ( parse_relion5_star_data( args.relion5_tomograms_star, args.tomogram, @@ -977,6 +977,7 @@ def match_template(argv=None): "minimal run requires tilt angles." ) voxel_size = args.voxel_size_angstrom + defocus_handedness = args.defocus_handedness tilt_angles = args.tilt_angles dose_accumulation = args.dose_accumulation per_tilt_weighting = args.per_tilt_weighting @@ -1014,7 +1015,7 @@ def match_template(argv=None): particle_diameter=args.particle_diameter, random_phase_correction=args.random_phase_correction, rng_seed=args.rng_seed, - defocus_handedness=args.defocus_handedness, + defocus_handedness=defocus_handedness, output_dtype=np.float16 if args.half_precision else np.float32, ) diff --git a/src/pytom_tm/io.py b/src/pytom_tm/io.py index 09c332e..a264e49 100644 --- a/src/pytom_tm/io.py +++ b/src/pytom_tm/io.py @@ -498,7 +498,7 @@ def parse_relion5_star_data( tomogram_path: pathlib.Path, phase_flip_correction: bool = False, phase_shift: float = 0.0, -) -> tuple[float, list[float, ...], list[float, ...], dict]: +) -> tuple[float, list[float, ...], list[float, ...], dict, int]: """Read RELION5 metadata from a project directory. Parameters @@ -514,8 +514,8 @@ def parse_relion5_star_data( Returns ------- - tomogram_voxel_size, tilt_angles, dose_accumulation, ctf_params: - tuple[float, list[float, ...], list[float, ...], list[dict, ...]] + tomogram_voxel_size, tilt_angles, dose_accumulation, ctf_params, defocus_handedness: + tuple[float, list[float, ...], list[float, ...], list[dict, ...], int] """ tomogram_id = tomogram_path.stem tomograms_star_data = starfile.read(tomograms_star_path) @@ -566,6 +566,7 @@ def parse_relion5_star_data( tomogram_meta_data["rlnTomoTiltSeriesPixelSize"] * tomogram_meta_data["rlnTomoTomogramBinning"] ) + defocus_handedness = int(tomogram_meta_data["rlnTomoHand"]) ctf_params = [ { @@ -584,4 +585,10 @@ def parse_relion5_star_data( * 1e-4 ] - return tomogram_voxel_size, tilt_angles, dose_accumulation, ctf_params + return ( + tomogram_voxel_size, + tilt_angles, + dose_accumulation, + ctf_params, + defocus_handedness, + ) diff --git a/tests/test_io.py b/tests/test_io.py index cc31a23..56a4777 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -92,12 +92,13 @@ def test_cast_warning(self): def test_parse_relion5_star_data(self): tomogram = pathlib.Path("rec_tomo200528_107.mrc") meta_data = parse_relion5_star_data(RELION5_TOMOGRAMS_STAR, tomogram) - self.assertEqual(len(meta_data), 4) + self.assertEqual(len(meta_data), 5) self.assertIsInstance(meta_data[0], float) self.assertIsInstance(meta_data[1], list) self.assertIsInstance(meta_data[2], list) self.assertIsInstance(meta_data[3], list) self.assertIsInstance(meta_data[3][0], dict) + self.assertIsInstance(meta_data[4], int) # test name mismatch tomogram = pathlib.Path("tomogram.mrc") From 4742a77f9aa1004923aee589f64021cdbfd253b9 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 18:44:49 +0200 Subject: [PATCH 11/18] update typing --- src/pytom_tm/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pytom_tm/io.py b/src/pytom_tm/io.py index a264e49..f320317 100644 --- a/src/pytom_tm/io.py +++ b/src/pytom_tm/io.py @@ -498,7 +498,7 @@ def parse_relion5_star_data( tomogram_path: pathlib.Path, phase_flip_correction: bool = False, phase_shift: float = 0.0, -) -> tuple[float, list[float, ...], list[float, ...], dict, int]: +) -> tuple[float, list[float, ...], list[float, ...], list[dict, ...], int]: """Read RELION5 metadata from a project directory. Parameters @@ -562,7 +562,7 @@ def parse_relion5_star_data( # _rlnTomoSizeY # 11 # _rlnTomoSizeZ # 12 # _rlnTomoReconstructedTomogram # 13 - tomogram_voxel_size = ( + tomogram_voxel_size = float( tomogram_meta_data["rlnTomoTiltSeriesPixelSize"] * tomogram_meta_data["rlnTomoTomogramBinning"] ) From 7bc90f7c7644e89aced1c3d5a2562713313e5dee Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 18:46:54 +0200 Subject: [PATCH 12/18] be helpful --- src/pytom_tm/entry_points.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pytom_tm/entry_points.py b/src/pytom_tm/entry_points.py index 68c64bc..9cf12f7 100644 --- a/src/pytom_tm/entry_points.py +++ b/src/pytom_tm/entry_points.py @@ -903,7 +903,11 @@ def match_template(argv=None): type=pathlib.Path, action=CheckFileExists, required=False, - help="TO-DO", + help="Here, you can provide a path to a RELION5 tomograms.star file (for " + "example " + "from a tomogram reconstruction job). pytom-match-pick will fetch all " + "the tilt-series metadata from this file and overwrite all other " + "metadata options.", ) device_group = parser.add_argument_group("Device control") device_group.add_argument( From 497d3f2ad566178969d40d126d06a7a1ab3d9d4a Mon Sep 17 00:00:00 2001 From: McHaillet Date: Tue, 15 Oct 2024 18:49:22 +0200 Subject: [PATCH 13/18] simplofy comment --- src/pytom_tm/io.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pytom_tm/io.py b/src/pytom_tm/io.py index eedca14..583c43d 100644 --- a/src/pytom_tm/io.py +++ b/src/pytom_tm/io.py @@ -524,8 +524,7 @@ def parse_relion5_star_data( "Multiple or zero matches of tomogram id in RELION5 STAR file. Aborting..." ) - # grab the path to tilt series star where tilt angles, defocus and dose are - # annotated + # grab the path to tilt series star for tilt angles, defocus and dose tilt_series_star_path = pathlib.Path( tomogram_meta_data["rlnTomoTiltSeriesStarFile"] ) From 96482c1cb52aa6e36be9969dbe6303c832ac2fc7 Mon Sep 17 00:00:00 2001 From: Marten Chaillet <58044494+McHaillet@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:57:12 +0200 Subject: [PATCH 14/18] update version number because of command line input change --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eed5a1a..f42f767 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pytom-match-pick" -version = "0.7.3" +version = "0.8.0" description = "PyTOM's GPU template matching module as an independent package" readme = "README.md" license = {file = "LICENSE"} From e1f12d927ae55710d40e66bf46118505e12a63bc Mon Sep 17 00:00:00 2001 From: Marten Chaillet <58044494+McHaillet@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:07:55 +0200 Subject: [PATCH 15/18] Apply suggestions from code review Co-authored-by: Sander Roet --- pyproject.toml | 2 +- src/pytom_tm/io.py | 24 +++++++----------------- tests/test_io.py | 14 +++++++++++++- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f42f767..76699ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pytom-match-pick" -version = "0.8.0" +version = "0.7.4" description = "PyTOM's GPU template matching module as an independent package" readme = "README.md" license = {file = "LICENSE"} diff --git a/src/pytom_tm/io.py b/src/pytom_tm/io.py index 583c43d..daa25b4 100644 --- a/src/pytom_tm/io.py +++ b/src/pytom_tm/io.py @@ -513,15 +513,18 @@ def parse_relion5_star_data( # match the tomo_id and check if viable matches = [ - (i, x) + i for i, x in enumerate(tomograms_star_data["rlnTomoName"]) - if x in tomogram_id + if tomogram_id.endswith(x) ] if len(matches) == 1: - tomogram_meta_data = tomograms_star_data.loc[matches[0][0]] + tomogram_meta_data = tomograms_star_data.loc[matches[0]] else: raise ValueError( - "Multiple or zero matches of tomogram id in RELION5 STAR file. Aborting..." + f"{'Multiple' if len(matches) > 1 else 'Zero'} matches " + f"of tomogram id: {tomogram_id}, " + f"in RELION5 STAR file: {tomograms_star_path}. " + "Aborting..." ) # grab the path to tilt series star for tilt angles, defocus and dose @@ -539,19 +542,6 @@ def parse_relion5_star_data( tilt_angles = list(tilt_series_star_data["rlnTomoNominalStageTiltAngle"]) dose_accumulation = list(tilt_series_star_data["rlnMicrographPreExposure"]) - # _rlnTomoName # 1 - # _rlnVoltage # 2 - # _rlnSphericalAberration # 3 - # _rlnAmplitudeContrast # 4 - # _rlnMicrographOriginalPixelSize # 5 - # _rlnTomoHand # 6 - # _rlnTomoTiltSeriesPixelSize # 7 - # _rlnTomoTiltSeriesStarFile # 8 - # _rlnTomoTomogramBinning # 9 - # _rlnTomoSizeX # 10 - # _rlnTomoSizeY # 11 - # _rlnTomoSizeZ # 12 - # _rlnTomoReconstructedTomogram # 13 tomogram_voxel_size = float( tomogram_meta_data["rlnTomoTiltSeriesPixelSize"] * tomogram_meta_data["rlnTomoTomogramBinning"] diff --git a/tests/test_io.py b/tests/test_io.py index 56a4777..678cd95 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -100,9 +100,21 @@ def test_parse_relion5_star_data(self): self.assertIsInstance(meta_data[3][0], dict) self.assertIsInstance(meta_data[4], int) - # test name mismatch tomogram = pathlib.Path("tomogram.mrc") with self.assertRaises( ValueError, msg="Unmatching tomograms name should raise an error." ): parse_relion5_star_data(RELION5_TOMOGRAMS_STAR, tomogram) + + tomogram = pathlib.Path("rec_tomogram200528_1077.mrc") + with self.assertRaises( + ValueError, msg="Partially matching tomogram name should raise an error." + ): + parse_relion5_star_data(RELION5_TOMOGRAMS_STAR, tomogram) + + + tomogram = pathlib.Path("rec_tomogram200528_10.mrc") + with self.assertRaises( + ValueError, msg="Partially matching tomogram name should raise an error." + ): + parse_relion5_star_data(RELION5_TOMOGRAMS_STAR, tomogram) From 871ceea1057ab3133ea90e6db517a3d1c8f99e94 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Wed, 16 Oct 2024 15:15:28 +0200 Subject: [PATCH 16/18] to the defocus multiplication in one go --- src/pytom_tm/io.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pytom_tm/io.py b/src/pytom_tm/io.py index daa25b4..08b9006 100644 --- a/src/pytom_tm/io.py +++ b/src/pytom_tm/io.py @@ -550,19 +550,17 @@ def parse_relion5_star_data( ctf_params = [ { - "defocus": defocus * 1e-6, + "defocus": defocus * 1e-10, "amplitude_contrast": tomogram_meta_data["rlnAmplitudeContrast"], "voltage": tomogram_meta_data["rlnVoltage"] * 1e3, "spherical_aberration": tomogram_meta_data["rlnSphericalAberration"] * 1e-3, "flip_phase": phase_flip_correction, - "phase_shift_deg": phase_shift, # Unclear to me where the phase - # shifts are annotated in RELION5 tilt series file + "phase_shift_deg": phase_shift, # RELION5 does not seem to store this } for defocus in ( tilt_series_star_data.rlnDefocusV + tilt_series_star_data.rlnDefocusU ) / 2 - * 1e-4 ] return ( From a97e83cfaab123fa28c780067cd993e33a0f45c0 Mon Sep 17 00:00:00 2001 From: McHaillet Date: Wed, 16 Oct 2024 15:39:21 +0200 Subject: [PATCH 17/18] make sure junk name removal is hit --- tests/test_tmjob.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_tmjob.py b/tests/test_tmjob.py index ad0ab17..64fb98d 100644 --- a/tests/test_tmjob.py +++ b/tests/test_tmjob.py @@ -613,7 +613,19 @@ def test_tm_job_half_precision(self): self.assertEqual(a.dtype, np.float32) def test_extractions(self): - _ = self.job.start_job(0, return_volumes=True) + self.job.tomo_id = "rec_" + self.job.tomo_id + scores, angles = self.job.start_job(0, return_volumes=True) + # set the appropriate headers when writing! + write_mrc( + TEST_DATA_DIR.joinpath(f"{self.job.tomo_id}_scores.mrc"), + scores, + self.job.voxel_size, + ) + write_mrc( + TEST_DATA_DIR.joinpath(f"{self.job.tomo_id}_angles.mrc"), + angles, + self.job.voxel_size, + ) # extract particles after running the job df, scores = extract_particles(self.job, 5, 100, create_plot=False) @@ -646,6 +658,7 @@ def test_extractions(self): msg="relion5 compat mode should return a centered " "location of the object", ) + self.assertNotIn("rec_", df_rel5["rlnTomoName"][0]) # test extraction mask that does not cover the particle df, scores = extract_particles( From c9ffc0249024d472aad58d0b62b3aa6f60be250a Mon Sep 17 00:00:00 2001 From: McHaillet Date: Wed, 16 Oct 2024 15:44:53 +0200 Subject: [PATCH 18/18] run test_io through ruff --- tests/test_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_io.py b/tests/test_io.py index 678cd95..1a0608b 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -112,7 +112,6 @@ def test_parse_relion5_star_data(self): ): parse_relion5_star_data(RELION5_TOMOGRAMS_STAR, tomogram) - tomogram = pathlib.Path("rec_tomogram200528_10.mrc") with self.assertRaises( ValueError, msg="Partially matching tomogram name should raise an error."