From f2bafcb5efb7f9c5507ab979438c238ded81cdb3 Mon Sep 17 00:00:00 2001 From: James Date: Tue, 22 Jun 2021 13:35:02 +0100 Subject: [PATCH 1/7] incorporate global masking for autoplotter plots --- velociraptor-plot | 2 + velociraptor/__init__.py | 2 +- velociraptor/autoplotter/objects.py | 89 +++++++++++++------------- velociraptor/catalogue/catalogue.py | 3 +- velociraptor/catalogue/registration.py | 15 ++++- 5 files changed, 62 insertions(+), 49 deletions(-) diff --git a/velociraptor-plot b/velociraptor-plot index f435b4c..b6d1ecf 100755 --- a/velociraptor-plot +++ b/velociraptor-plot @@ -137,6 +137,8 @@ if __name__ == "__main__": style.use(args.stylesheet) print_if_debug(f"Generating initial AutoPlotter instance for {args.config}.") + print(args.config) + quit() auto_plotter = AutoPlotter(args.config) print_if_debug(f"Loading halo catalogue at {args.properties}.") if args.registration is not None: diff --git a/velociraptor/__init__.py b/velociraptor/__init__.py index 71a3ea4..c3710e5 100644 --- a/velociraptor/__init__.py +++ b/velociraptor/__init__.py @@ -76,5 +76,5 @@ def load( if registration_file_path is not None: catalogue.register_derived_quantities(registration_file_path) - + return catalogue diff --git a/velociraptor/autoplotter/objects.py b/velociraptor/autoplotter/objects.py index c26c7ab..177448a 100644 --- a/velociraptor/autoplotter/objects.py +++ b/velociraptor/autoplotter/objects.py @@ -11,7 +11,7 @@ from unyt import unyt_quantity, unyt_array, matplotlib_support from unyt.exceptions import UnitConversionError -from numpy import log10, linspace, logspace, array, logical_and +from numpy import log10, linspace, logspace, array, logical_and, ones from matplotlib.pyplot import Axes, Figure, close from yaml import safe_load from typing import Union, List, Dict, Tuple @@ -95,6 +95,8 @@ class VelociraptorPlot(object): observational_data_filenames: List[str] observational_data_bracket_width: float observational_data_directory: str + # global mask + global_mask: Union[None, array] def __init__( self, @@ -743,7 +745,7 @@ def _add_lines_to_axes(self, ax: Axes, x: unyt_array, y: unyt_array) -> None: return def get_quantity_from_catalogue_with_mask( - self, quantity: str, catalogue: VelociraptorCatalogue + self, quantity: str, catalogue: VelociraptorCatalogue, ) -> unyt_array: """ Get a quantity from the catalogue using the mask. @@ -754,61 +756,51 @@ def get_quantity_from_catalogue_with_mask( # in versions of unyt less than 2.6.0 name = x.name + # temporary masked array to apply global mask + # in concert with plot-specific masks + x_mask = self.global_mask + if self.structure_mask is not None: - x = x[self.structure_mask] + # if structure_mask already set, mask and return + x_mask = logical_and(x_mask, self.structure_mask) + x = x[x_mask] x.name = name - elif self.selection_mask is not None: + return x + + # allow all entries by default + self.structure_mask = ones(x.shape).astype(bool) + + if self.selection_mask is not None: # Create mask self.structure_mask = reduce( getattr, self.selection_mask.split("."), catalogue ).astype(bool) - - if self.select_structure_type is not None: - if self.select_structure_type == self.exclude_structure_type: - raise AutoPlotterError( - f"Cannot simultaneously select and exclude structure" - " type {self.select_structure_type}" - ) - self.structure_mask = logical_and( - self.structure_mask, - catalogue.structure_type.structuretype - == self.select_structure_type, - ) - - elif self.exclude_structure_type is not None: - self.structure_mask = logical_and( - self.structure_mask, - catalogue.structure_type.structuretype - != self.exclude_structure_type, - ) - - x = x[self.structure_mask] - x.name = name - elif self.select_structure_type is not None: + if self.select_structure_type is not None: if self.select_structure_type == self.exclude_structure_type: raise AutoPlotterError( f"Cannot simultaneously select and exclude structure" " type {self.select_structure_type}" ) - - # Need to create mask - self.structure_mask = ( - catalogue.structure_type.structuretype == self.select_structure_type + self.structure_mask = logical_and( + self.structure_mask, + catalogue.structure_type.structuretype + == self.select_structure_type, ) - - x = x[self.structure_mask] - x.name = name - elif self.exclude_structure_type is not None: - # Need to create mask - self.structure_mask = ( - catalogue.structure_type.structuretype != self.exclude_structure_type + if self.exclude_structure_type is not None: + self.structure_mask = logical_and( + self.structure_mask, + catalogue.structure_type.structuretype + != self.exclude_structure_type, ) + + # combine global and structure masks + x_mask = logical_and(x_mask, self.structure_mask) - x = x[self.structure_mask] - x.name = name - + # apply to the unyt array of values + x = x[x_mask] + x.name = name return x - + def _make_plot_scatter( self, catalogue: VelociraptorCatalogue ) -> Tuple[Figure, Axes]: @@ -974,7 +966,7 @@ def _make_plot_cumulative_histogram( return fig, ax def make_plot( - self, catalogue: VelociraptorCatalogue, directory: str, file_extension: str + self, catalogue: VelociraptorCatalogue, directory: str, file_extension: str, ): """ Federates out data parsing to individual functions based on the @@ -1058,7 +1050,9 @@ class AutoPlotter(object): observational_data_directory: str # Whether or not the plots were created successfully. created_successfully: List[bool] - + # global mask + global_mask: Union[None, array] + def __init__( self, filename: Union[str, List[str]], @@ -1123,7 +1117,7 @@ def parse_yaml(self): return - def link_catalogue(self, catalogue: VelociraptorCatalogue): + def link_catalogue(self, catalogue: VelociraptorCatalogue, global_mask_tag: str): """ Links a catalogue with this object so that the plots can actually be created. @@ -1131,6 +1125,10 @@ def link_catalogue(self, catalogue: VelociraptorCatalogue): self.catalogue = catalogue + if global_mask_tag is not None: + self.global_mask = reduce(getattr, quantity.split("."), catalogue) + else: + self.global_mask = True return def create_plots( @@ -1150,6 +1148,7 @@ def create_plots( for plot in self.plots: try: + plot.global_mask = self.global_mask plot.make_plot( catalogue=self.catalogue, directory=directory, diff --git a/velociraptor/catalogue/catalogue.py b/velociraptor/catalogue/catalogue.py index 03135a7..2d7a4bc 100644 --- a/velociraptor/catalogue/catalogue.py +++ b/velociraptor/catalogue/catalogue.py @@ -1,4 +1,4 @@ -""" +b""" Main objects for the velociraptor reading library. This is based upon the reading routines in the SWIFTsimIO library. @@ -412,4 +412,3 @@ def register_derived_quantities(self, registration_file_path: str) -> None: self.derived_quantities = DerivedQuantities(registration_file_path, self) return - diff --git a/velociraptor/catalogue/registration.py b/velociraptor/catalogue/registration.py index fd0b187..9ebdc62 100644 --- a/velociraptor/catalogue/registration.py +++ b/velociraptor/catalogue/registration.py @@ -37,7 +37,8 @@ def registration_fail_all( + name: A fancy (possibly LaTeX'd) name for the field. + snake_case: A correct snake_case name for the field. """ - + + if field_path == "ThisFieldPathWouldNeverExist": return ( unit_system.length, @@ -1442,7 +1443,18 @@ def registration_spherical_overdensities( else: raise RegistrationDoesNotMatchError +def registration_bgpart_masses( + field_path: str, unit_system: VelociraptorUnits +) -> (unyt.Unit, str, str): + """ + Registers the stellar ages properties (currently tage_star). + """ + if field_path == "Mass_interloper": + return unit_system.mass, "Mass from background particles", "bgpart_masses" + else: + raise RegistrationDoesNotMatchError + # TODO # lambda_B # q @@ -1502,6 +1514,7 @@ def registration_spherical_overdensities( "cold_dense_gas_properties", "log_element_ratios_times_masses", "lin_element_ratios_times_masses", + "bgpart_masses", "fail_all", ] } From 44f7df166b97acd8c4c38c61ca22296f8d94a16f Mon Sep 17 00:00:00 2001 From: James Date: Tue, 22 Jun 2021 14:06:18 +0100 Subject: [PATCH 2/7] fix global mask parsing --- velociraptor/autoplotter/objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/velociraptor/autoplotter/objects.py b/velociraptor/autoplotter/objects.py index 177448a..1baf8a8 100644 --- a/velociraptor/autoplotter/objects.py +++ b/velociraptor/autoplotter/objects.py @@ -1126,7 +1126,7 @@ def link_catalogue(self, catalogue: VelociraptorCatalogue, global_mask_tag: str) self.catalogue = catalogue if global_mask_tag is not None: - self.global_mask = reduce(getattr, quantity.split("."), catalogue) + self.global_mask = reduce(getattr, global_mask_tag.split("."), catalogue) else: self.global_mask = True return From 12cdf1b5f931b5cbbaf3db612d3bf3bc3a46f106 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 25 Jun 2021 15:59:59 +0100 Subject: [PATCH 3/7] add fixes from matthieu's review --- velociraptor-plot | 2 -- velociraptor/catalogue/catalogue.py | 2 +- velociraptor/catalogue/registration.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/velociraptor-plot b/velociraptor-plot index b6d1ecf..f435b4c 100755 --- a/velociraptor-plot +++ b/velociraptor-plot @@ -137,8 +137,6 @@ if __name__ == "__main__": style.use(args.stylesheet) print_if_debug(f"Generating initial AutoPlotter instance for {args.config}.") - print(args.config) - quit() auto_plotter = AutoPlotter(args.config) print_if_debug(f"Loading halo catalogue at {args.properties}.") if args.registration is not None: diff --git a/velociraptor/catalogue/catalogue.py b/velociraptor/catalogue/catalogue.py index 2d7a4bc..27d71c4 100644 --- a/velociraptor/catalogue/catalogue.py +++ b/velociraptor/catalogue/catalogue.py @@ -1,4 +1,4 @@ -b""" +""" Main objects for the velociraptor reading library. This is based upon the reading routines in the SWIFTsimIO library. diff --git a/velociraptor/catalogue/registration.py b/velociraptor/catalogue/registration.py index 9ebdc62..63b766c 100644 --- a/velociraptor/catalogue/registration.py +++ b/velociraptor/catalogue/registration.py @@ -1447,7 +1447,7 @@ def registration_bgpart_masses( field_path: str, unit_system: VelociraptorUnits ) -> (unyt.Unit, str, str): """ - Registers the stellar ages properties (currently tage_star). + Registers the halo mass contributed by background particle interlopers. """ if field_path == "Mass_interloper": return unit_system.mass, "Mass from background particles", "bgpart_masses" From cd3f7fab96de022c7d3a35b60e814727eaf65e76 Mon Sep 17 00:00:00 2001 From: James Date: Mon, 5 Jul 2021 12:54:17 +0100 Subject: [PATCH 4/7] implement evgeniis review --- velociraptor/autoplotter/objects.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/velociraptor/autoplotter/objects.py b/velociraptor/autoplotter/objects.py index 1baf8a8..8cf7c95 100644 --- a/velociraptor/autoplotter/objects.py +++ b/velociraptor/autoplotter/objects.py @@ -758,11 +758,11 @@ def get_quantity_from_catalogue_with_mask( # temporary masked array to apply global mask # in concert with plot-specific masks - x_mask = self.global_mask + # x_mask = self.global_mask if self.structure_mask is not None: # if structure_mask already set, mask and return - x_mask = logical_and(x_mask, self.structure_mask) + x_mask = logical_and(self.global_mask, self.structure_mask) x = x[x_mask] x.name = name return x @@ -794,7 +794,7 @@ def get_quantity_from_catalogue_with_mask( ) # combine global and structure masks - x_mask = logical_and(x_mask, self.structure_mask) + x_mask = logical_and(self.global_mask, self.structure_mask) # apply to the unyt array of values x = x[x_mask] @@ -1117,7 +1117,7 @@ def parse_yaml(self): return - def link_catalogue(self, catalogue: VelociraptorCatalogue, global_mask_tag: str): + def link_catalogue(self, catalogue: VelociraptorCatalogue, global_mask_tag: Union[None, str]): """ Links a catalogue with this object so that the plots can actually be created. From 7312d1377e1c750a03a6e8334c81d7ae39e8bb76 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 14 Jul 2021 12:15:25 +0100 Subject: [PATCH 5/7] integrate mass interlopers into "masses" structure --- velociraptor/catalogue/registration.py | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/velociraptor/catalogue/registration.py b/velociraptor/catalogue/registration.py index 63b766c..de26721 100644 --- a/velociraptor/catalogue/registration.py +++ b/velociraptor/catalogue/registration.py @@ -308,6 +308,8 @@ def registration_masses( full_name = "$M_{\\rm FOF}$" elif field_path == "Mass_tot": full_name = r"$M$" + elif field_path == "Mass_interloper": + full_name = "$M_{\\rm BG}$" # General regex matching case. @@ -322,7 +324,6 @@ def registration_masses( ) regex = cached_regex(match_string) match = regex.match(field_path) - if match and not full_name: mass = match.group(1) radius = match.group(2) @@ -1442,18 +1443,6 @@ def registration_spherical_overdensities( return unit, full_name, snake_case else: raise RegistrationDoesNotMatchError - -def registration_bgpart_masses( - field_path: str, unit_system: VelociraptorUnits -) -> (unyt.Unit, str, str): - """ - Registers the halo mass contributed by background particle interlopers. - """ - if field_path == "Mass_interloper": - return unit_system.mass, "Mass from background particles", "bgpart_masses" - else: - raise RegistrationDoesNotMatchError - # TODO # lambda_B @@ -1514,7 +1503,6 @@ def registration_bgpart_masses( "cold_dense_gas_properties", "log_element_ratios_times_masses", "lin_element_ratios_times_masses", - "bgpart_masses", "fail_all", ] } From 89acef8bb6a67d79327a74f834bcbd5a98e2d942 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 14 Jul 2021 13:44:51 +0100 Subject: [PATCH 6/7] add global_mask_tag=None to script --- velociraptor-plot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/velociraptor-plot b/velociraptor-plot index f435b4c..ec99072 100755 --- a/velociraptor-plot +++ b/velociraptor-plot @@ -149,7 +149,7 @@ if __name__ == "__main__": registration_file_path=args.registration, ) print_if_debug(f"Linking catalogue and AutoPlotter instance.") - auto_plotter.link_catalogue(catalogue=catalogue) + auto_plotter.link_catalogue(catalogue=catalogue, global_mask_tag=None) print_if_debug( f"Creating figures with extension .{args.file_type} in {args.output}." From 508c5a3b910c19ba1ac14ed554618e1dd7fe04bc Mon Sep 17 00:00:00 2001 From: James Date: Wed, 14 Jul 2021 16:41:38 +0100 Subject: [PATCH 7/7] remove commented lines --- velociraptor/autoplotter/objects.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/velociraptor/autoplotter/objects.py b/velociraptor/autoplotter/objects.py index 8cf7c95..786f23f 100644 --- a/velociraptor/autoplotter/objects.py +++ b/velociraptor/autoplotter/objects.py @@ -755,10 +755,6 @@ def get_quantity_from_catalogue_with_mask( # We give each dataset a custom name, that gets ruined when masking # in versions of unyt less than 2.6.0 name = x.name - - # temporary masked array to apply global mask - # in concert with plot-specific masks - # x_mask = self.global_mask if self.structure_mask is not None: # if structure_mask already set, mask and return