diff --git a/CI/unit_tests/data/test_double_well_potential.py b/CI/unit_tests/data/test_double_well_potential.py index 2967f36..bb9afbf 100644 --- a/CI/unit_tests/data/test_double_well_potential.py +++ b/CI/unit_tests/data/test_double_well_potential.py @@ -9,6 +9,7 @@ Description: Test the double_well_potential module. """ import unittest + from symsuite.data.double_well_potential import DoubleWellPotential @@ -52,5 +53,5 @@ def test_double_well(self): self.assertEqual(len(self.generator.image), 500) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/CI/unit_tests/distance_metrics/test_angular_distance.py b/CI/unit_tests/distance_metrics/test_angular_distance.py new file mode 100644 index 0000000..0d16ae3 --- /dev/null +++ b/CI/unit_tests/distance_metrics/test_angular_distance.py @@ -0,0 +1,72 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: +Summary +------- +Test the angular distance module. +""" +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import jax.numpy as np +from numpy.testing import assert_array_almost_equal + +from symsuite.distance_metrics.angular_distance import AngularDistance + + +class TestAngularDistance: + """ + Class to test the cosine distance measure module. + """ + + def test_angular_distance(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = AngularDistance() + + # Test orthogonal vectors + point_1 = np.array([[1, 0]]) + point_2 = np.array([[0, 1]]) + assert_array_almost_equal(metric(point_1, point_2), [0.5]) + + # Test parallel vectors + point_1 = np.array([[1, 0]]) + point_2 = np.array([[1, 1]]) + assert_array_almost_equal(metric(point_1, point_2), [0.25]) + + def test_multiple_distances(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = AngularDistance() + + # Test orthogonal vectors + point_1 = np.array([[1, 0], [1, 0]]) + point_2 = np.array([[0, 1], [1, 1]]) + assert_array_almost_equal(metric(point_1, point_2), [0.5, 0.25]) diff --git a/CI/unit_tests/distance_metrics/test_cosine_distance.py b/CI/unit_tests/distance_metrics/test_cosine_distance.py new file mode 100644 index 0000000..6f479a8 --- /dev/null +++ b/CI/unit_tests/distance_metrics/test_cosine_distance.py @@ -0,0 +1,75 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: +Summary +------- +Test the cosine distance module. +""" +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import jax.numpy as np +from numpy.testing import assert_array_almost_equal + +from symsuite.distance_metrics.cosine_distance import CosineDistance + + +class TestCosineDistance: + """ + Class to test the cosine distance measure module. + """ + + def test_cosine_distance(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = CosineDistance() + + # Test orthogonal vectors + point_1 = np.array([[1, 0, 0, 0]]) + point_2 = np.array([[0, 1, 0, 0]]) + assert_array_almost_equal(metric(point_1, point_2), [1]) + + # Test parallel vectors + assert_array_almost_equal(metric(point_1, point_1), [0]) + + # Somewhere in between + point_1 = np.array([[1.0, 0, 0, 0]]) + point_2 = np.array([[0.5, 1.0, 0, 3.0]]) + assert_array_almost_equal(metric(point_1, point_2), [0.84382623]) + + def test_multiple_distances(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = CosineDistance() + + # Test orthogonal vectors + point_1 = np.array([[1, 0, 0, 0], [1, 0, 0, 0], [1.0, 0, 0, 0]]) + point_2 = np.array([[0, 1, 0, 0], [1, 0, 0, 0], [0.5, 1.0, 0, 3.0]]) + assert_array_almost_equal(metric(point_1, point_2), [1, 0, 0.843826], decimal=6) diff --git a/CI/unit_tests/distance_metrics/test_hyper_sphere_distance.py b/CI/unit_tests/distance_metrics/test_hyper_sphere_distance.py new file mode 100644 index 0000000..aac3ff8 --- /dev/null +++ b/CI/unit_tests/distance_metrics/test_hyper_sphere_distance.py @@ -0,0 +1,87 @@ +""" +ZnRND: A Zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Test the hyper sphere distance module. +""" +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import jax.numpy as np +from numpy.testing import assert_array_almost_equal + +from symsuite.distance_metrics.hyper_sphere_distance import HyperSphere + + +class TestCosineDistance: + """ + Class to test the cosine distance measure module. + """ + + def test_hyper_sphere_distance(self): + """ + Test the hyper sphere distance. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = HyperSphere(order=2) + + # Test orthogonal vectors + point_1 = np.array([[1, 0, 0, 0]]) + point_2 = np.array([[0, 1, 0, 0]]) + assert_array_almost_equal(metric(point_1, point_2), [1.41421356]) + + # Test parallel vectors + point_1 = np.array([[1, 0, 0, 0]]) + point_2 = np.array([[1, 0, 0, 0]]) + assert_array_almost_equal(metric(point_1, point_2), [0]) + + # Somewhere in between + point_1 = np.array([[1.0, 0, 0, 0]]) + point_2 = np.array([[0.5, 1.0, 0, 3.0]]) + assert_array_almost_equal( + metric(point_1, point_2), [0.84382623 * np.sqrt(10.25)] + ) + + def test_multiple_distances(self): + """ + Test the hyper sphere distance. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = HyperSphere(order=2) + + # Test orthogonal vectors + point_1 = np.array([[1, 0, 0, 0], [1, 0, 0, 0], [1.0, 0, 0, 0]]) + point_2 = np.array([[0, 1, 0, 0], [1, 0, 0, 0], [0.5, 1.0, 0, 3.0]]) + assert_array_almost_equal( + metric(point_1, point_2), + [np.sqrt(2), 0, 0.84382623 * np.sqrt(10.25)], + decimal=6, + ) diff --git a/CI/unit_tests/distance_metrics/test_l_p_norm.py b/CI/unit_tests/distance_metrics/test_l_p_norm.py new file mode 100644 index 0000000..e142006 --- /dev/null +++ b/CI/unit_tests/distance_metrics/test_l_p_norm.py @@ -0,0 +1,85 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Test the l_p norm metric. +""" +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import jax.numpy as np +from numpy.testing import assert_almost_equal, assert_array_almost_equal + +from symsuite.distance_metrics.l_p_norm import LPNorm + + +class TestLPNorm: + """ + Class to test the cosine distance measure module. + """ + + def test_l_2_distance(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = LPNorm(order=2) + + # Test orthogonal vectors + point_1 = np.array([[1.0, 7.0, 0.0, 0.0]]) + point_2 = np.array([[1.0, 1.0, 0.0, 0.0]]) + + assert_array_almost_equal(metric(point_1, point_2), [6.0]) + + def test_l_3_distance(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = LPNorm(order=3) + + # Test orthogonal vectors + point_1 = np.array([[1.0, 7.0, 0.0, 0.0]]) + point_2 = np.array([[1.0, 1.0, 0.0, 0.0]]) + assert_almost_equal(metric(point_1, point_2), [6.0], decimal=4) + + def test_multi_distance(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = LPNorm(order=1) + + # Test orthogonal vectors + point_1 = np.array([[1.0, 7.0, 0.0, 0.0], [4, 7, 2, 1]]) + point_2 = np.array([[1.0, 1.0, 0.0, 0.0], [6, 3, 1, 8]]) + assert_array_almost_equal(metric(point_1, point_2), [6.0, 14.0], decimal=4) diff --git a/CI/unit_tests/distance_metrics/test_mahalanobis_distance.py b/CI/unit_tests/distance_metrics/test_mahalanobis_distance.py new file mode 100644 index 0000000..93f8e72 --- /dev/null +++ b/CI/unit_tests/distance_metrics/test_mahalanobis_distance.py @@ -0,0 +1,170 @@ +""" +ZnRND: A Zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Test the angular distance module. +""" +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import jax +import jax.numpy as np +import numpy as onp +import scipy.spatial.distance +from numpy.testing import assert_almost_equal, assert_array_almost_equal + +from symsuite.distance_metrics.mahalanobis_distance import MahalanobisDistance + + +class TestMahalanobisDistance: + """ + Class to test the cosine distance measure module. + """ + + @classmethod + def setup_class(cls): + """ + Prepare the test suite. + """ + cls.key = jax.random.PRNGKey(0) + + def test_mahalanobis_distance(self): + """ + Test the Mahalanobis distance on functionality by comparing results a + test Mahalanobis distance from scipy. + + Returns + ------- + Assert if the Mahalanobis distance returns true values for sample set of + random normal distributed points in two dimensions. + """ + metric = MahalanobisDistance() + + # Create sample set + point_1, point_2 = self.create_sample_set() + + # Calculate results from distance metric + metric_results = metric(np.array(point_1), np.array(point_2)) + + # Calculate test results from numpy distance metric + test_metric_results = [] + self.calculate_numpy_mahalanobis_distance(point_1, point_2, test_metric_results) + + # Assert results + assert_almost_equal(metric_results, test_metric_results, decimal=1) + + def test_identity(self): + """ + Test the identity criterion of a metric, based on a randomly produced sample + set (used to create the covariance matrix). + + Returns + ------- + Asserts if the distance of the last point of point_1 and point_2 is equal to 0 + """ + # Create Sample set + point_1, point_2 = self.create_sample_set() + + # Add point of interest + point_of_interest = np.array([[7.0, 3.0]]) + point_1 = np.concatenate([np.array(point_1), point_of_interest], axis=0) + point_2 = np.concatenate([np.array(point_2), point_of_interest], axis=0) + + # Assert identity + metric = MahalanobisDistance() + assert_array_almost_equal(metric(point_1, point_2)[-1], 0) + + def test_symmetry(self): + """ + Test the symmetry criterion of a metric, based on a randomly produced sample + set (used to create the covariance matrix). + + Returns + ------- + Asserts if the distances of the last two points of point_1 and point_2 are + identical. + """ + # Create Sample set + point_1, point_2 = self.create_sample_set() + + # Add point of interest + point_1_of_interest = np.array([[-2.0, 5.0], [7.0, 3.0]]) + point_2_of_interest = np.array([[7.0, 3.0], [-2.0, 5.0]]) + point_1 = np.concatenate( + [np.array(point_1), point_1_of_interest], + axis=0, + ) + point_2 = np.concatenate( + [np.array(point_2), point_2_of_interest], + axis=0, + ) + + # Assert identity + metric = MahalanobisDistance() + assert_array_almost_equal( + metric(point_1, point_2)[-1], (metric(point_1, point_2)[-2]) + ) + + @staticmethod + def create_sample_set(): + """ + + Returns + ------- + Creates a random normal distributed sample set + """ + point_1 = np.array( + [onp.random.normal(0, 10, 100), onp.random.normal(0, 20, 100)] + ).T + point_2 = np.array( + [onp.random.normal(0, 10, 100), onp.random.normal(0, 20, 100)] + ).T + return point_1, point_2 + + @staticmethod + def calculate_numpy_mahalanobis_distance( + point_1: np.ndarray, point_2: np.ndarray, result_list: list + ): + """ + Calculates the Mahalanobis distance based on a scipy integration. + + Parameters + ---------- + point_1 : np.ndarray + Set of points in the distance calculation. + point_2 : np.ndarray + Set of points in the distance calculation. + result_list : list + Results for each point are appended to this list. + + Returns + ------- + Appends all calculated distances to the result_list. + """ + inv_cov = np.linalg.inv(np.cov(point_1.T)) + for index in range(len(point_1.T[0, :])): + result_list.append( + scipy.spatial.distance.mahalanobis( + point_1[index], point_2[index], inv_cov + ) + ) diff --git a/CI/unit_tests/distance_metrics/test_order_n_difference.py b/CI/unit_tests/distance_metrics/test_order_n_difference.py new file mode 100644 index 0000000..e7340ec --- /dev/null +++ b/CI/unit_tests/distance_metrics/test_order_n_difference.py @@ -0,0 +1,85 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Test the order n norm metric. +""" +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import jax.numpy as np +from numpy.testing import assert_almost_equal, assert_array_equal + +from symsuite.distance_metrics.order_n_difference import OrderNDifference + + +class TestOrderNDifference: + """ + Class to test the cosine distance measure module. + """ + + def test_order_2_distance(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = OrderNDifference(order=2, reduce_operation="sum") + + # Test orthogonal vectors + point_1 = np.array([[1.0, 7.0, 0.0, 0.0]]) + point_2 = np.array([[1.0, 1.0, 0.0, 0.0]]) + assert_array_equal(metric(point_1, point_2), [36.0]) + + def test_order_3_distance(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = OrderNDifference(order=3, reduce_operation="sum") + + # Test orthogonal vectors + point_1 = np.array([[1.0, 1.0, 0.0, 0.0]]) + point_2 = np.array([[1.0, 7.0, 0.0, 0.0]]) + + assert_almost_equal(metric(point_1, point_2), [-216.0], decimal=4) + + def test_multi_distance(self): + """ + Test the cosine similarity measure. + + Returns + ------- + Assert the correct answer is returned for orthogonal, parallel, and + somewhere in between. + """ + metric = OrderNDifference(order=3, reduce_operation="sum") + + # Test orthogonal vectors + point_1 = np.array([[1.0, 7.0, 0.0, 0.0], [4, 7, 2, 1]]) + point_2 = np.array([[1.0, 1.0, 0.0, 0.0], [6, 3, 1, 8]]) + assert_almost_equal(metric(point_1, point_2), [216.0, -286.0], decimal=4) diff --git a/CI/unit_tests/loss_functions/test_loss_functions.py b/CI/unit_tests/loss_functions/test_loss_functions.py new file mode 100644 index 0000000..1dfc70c --- /dev/null +++ b/CI/unit_tests/loss_functions/test_loss_functions.py @@ -0,0 +1,93 @@ +""" +ZnRND: A Zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Module for testing the loss functions + +Notes +----- +As the loss functions come directly from distance metrics and the distance metrics are +heavily tested, here we test all loss functions on the same set of data and ensure that +the results are as expected. +""" +import os + +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import jax.numpy as np +import pytest + +from symsuite.loss_functions import ( + AngleDistanceLoss, + CosineDistanceLoss, + LPNormLoss, + MeanPowerLoss, +) + + +class TestLossFunctions: + """ + Class for the testing of the ZnRND loss functions. + """ + + @classmethod + def setup_class(cls): + """ + Prepare the test class + """ + cls.linear_predictions = np.array([[1, 1, 2], [9, 9, 9], [0, 0, 0], [9, 1, 1]]) + cls.linear_targets = np.array([[9, 9, 9], [1, 1, 2], [9, 1, 1], [0, 0, 0]]) + + cls.angular_predictions = np.array([[0, 0, 1], [1, 0, 0], [1, 1, 0], [1, 0, 1]]) + cls.angular_targets = np.array([[1, 0, 0], [0, 0, 1], [1, 0, 1], [1, 1, 0]]) + + def test_absolute_angle(self): + """ + Test the absolute angle loss + """ + loss = AngleDistanceLoss()( + self.angular_predictions / 9, self.angular_targets / 9 + ) + loss == pytest.approx(0.417, 0.0001) + + def test_cosine_distance(self): + """ + Test the cosine_distance loss + """ + loss = CosineDistanceLoss()( + self.angular_predictions / 9, self.angular_targets / 9 + ) + loss == 0.75 + + def test_l_p_norm(self): + """ + Test the l_p norm loss + """ + loss = LPNormLoss(order=2)(self.linear_predictions, self.linear_targets) + loss == pytest.approx(11.207, 0.0001) + + def test_mean_power(self): + """ + Test the mean_power loss + """ + loss = MeanPowerLoss(order=2)(self.linear_predictions, self.linear_targets) + loss == 130.0 diff --git a/CI/unit_tests/symmetry_groups/test_data_clustering.py b/CI/unit_tests/symmetry_groups/test_data_clustering.py index 931ef42..280bab1 100644 --- a/CI/unit_tests/symmetry_groups/test_data_clustering.py +++ b/CI/unit_tests/symmetry_groups/test_data_clustering.py @@ -2,6 +2,7 @@ Test module for the Data Clustering module. """ import unittest + import numpy as np diff --git a/docs/source/conf.py b/docs/source/conf.py index 9fe9555..e70eb26 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -3,6 +3,7 @@ """ import os import sys + import sphinx_rtd_theme sys.path.insert(0, os.path.abspath(".")) @@ -47,35 +48,28 @@ # Material theme options (see theme.conf for more information) html_theme_options = { - # Set the name of the project to appear in the navigation. - 'nav_title': 'SymDet', - + "nav_title": "SymDet", # Set you GA account ID to enable tracking - 'google_analytics_account': 'UA-XXXXX', - + "google_analytics_account": "UA-XXXXX", # Specify a base_url used to generate sitemap.xml. If not # specified, then no sitemap will be built. - 'base_url': 'https://symdet.readthedocs.io/en/latest/', - + "base_url": "https://symdet.readthedocs.io/en/latest/", # Set the color and the accent color - 'color_primary': 'blue', - 'color_accent': 'light-blue', - + "color_primary": "blue", + "color_accent": "light-blue", # Set the repo location to get a badge with stats - 'repo_url': 'https://github.com/SamTov/SymDet', - 'repo_name': 'SymDet', - + "repo_url": "https://github.com/SamTov/SymDet", + "repo_name": "SymDet", # Visible levels of the global TOC; -1 means unlimited - 'globaltoc_depth': 3, + "globaltoc_depth": 3, # If False, expand all TOC entries - 'globaltoc_collapse': False, + "globaltoc_collapse": False, # If True, show hidden TOC entries - 'globaltoc_includehidden': False, + "globaltoc_includehidden": False, } - # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/examples/notebooks/SO_example.ipynb b/examples/notebooks/SO_example.ipynb index 8d02ffe..798b5c8 100644 --- a/examples/notebooks/SO_example.ipynb +++ b/examples/notebooks/SO_example.ipynb @@ -344,7 +344,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -358,7 +358,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" + "version": "3.8.12" } }, "nbformat": 4, diff --git a/examples/notebooks/double_well_investigation.ipynb b/examples/notebooks/double_well_investigation.ipynb index 9e07240..4bbda77 100644 --- a/examples/notebooks/double_well_investigation.ipynb +++ b/examples/notebooks/double_well_investigation.ipynb @@ -37,9 +37,18 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/samueltovey/miniconda3/envs/zincware/lib/python3.8/site-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.16.5 and <1.23.0 is required for this version of SciPy (detected version 1.23.1\n", + " warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n" + ] + } + ], "source": [ - "import symdet" + "import symsuite" ] }, { @@ -56,7 +65,7 @@ "metadata": {}, "outputs": [], "source": [ - "double_well_potential = symdet.DoubleWellPotential(a=2.4)" + "double_well_potential = symsuite.DoubleWellPotential(a=2.4)" ] }, { @@ -71,9 +80,16 @@ "execution_count": 3, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:absl:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" + ] + }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmWElEQVR4nO3de3zU9Z3v8dcnCQERqhFckFKCqeiisAWDBK3LhlVawB6BQneRrra7WKpbzzndds82Rddaaind3e6y+9ADXrfSitiqIKso3qCpi0QIYrkVwZTEeGkFBg4XIbfP+WMmmMvMkMBkfr+ZvJ+PRx7M78LkncvkM7/v9/f9fs3dERERSSQn6AAiIhJuKhQiIpKUCoWIiCSlQiEiIkmpUIiISFJ5QQfoCv379/ehQ4e22nf06FHOPvvsYAKdgrJ1XlhzgbKdrrBmC2suSG22ysrKfe5+ftyD7p51H8XFxd7W2rVr2+0LC2XrvLDmcle20xXWbGHN5Z7abMAmT/A3VU1PIiKSlAqFiIgkpUIhIiJJqVCIiEhSKhQiIpKUCoWIiCSlQiEiIkmpUIiISFIqFCIikpQKhYiIJKVCISIiSalQiIhIUioUIiKSlAqFiIgkpUIhIiJJqVCIiEhSKhQiIpKUCoWIiCQVaKEws4fN7A9mti3BcTOz/zCzPWb2GzO7PN0ZRUS6u6CvKH4KTEpyfDIwLPYxF1ichkwiItJCoIXC3cuBA0lOmQosja39vQE418wuSE86EZHsUVkd4d61e6isjnT6/+Z1QZ5U+iTwTovt2ti+94OJIyKSeSqrI3z5wQ3UNTSRn5fDozePo7iwoMP/39y9C+N1IIDZUOAZdx8R59gzwEJ3fzW2/TLwHXffFOfcuUSbpxgwYEDx8uXLWx0/cuQIffr0Sf0XkALK1nlhzQXKdrrCmi2suaDj2Z55u44nd9fjRJuRvjisB1/4dH6rcyZMmFDp7mPiPoG7B/oBDAW2JTh2H3BDi+1dwAWnes7i4mJva+3ate32hYWydV5Yc7kr2+kKa7aw5nLveLZNew/4JXes9qKyZ/ySO1b7pr0H2p0DbPIEf1PD3vS0CrjNzJYDJcAhd1ezk4hIJxQXFvDozePYULWfcUX9OtXsBAH3UZjZY0Ap0N/MaoHvAT0A3H0JsBqYAuwBjgF/HUxSEZHMVVkdOe0iAQEXCne/4RTHHfhGmuKIiGSdM+3IhuDHUYiISBfaULWfuoYmmhzqG5rYULW/08+hQiEiksXGFfUjPy+HXIMeeTmMK+rX6ecIe2e2iIicgTPtyAYVChGRrLWsoobntr3P5BEX8I0JF53286hQiIhkoWUVNcxbsRWAX+/eB8DskiGn9VzqoxARyULPbXs/6XZnqFCIiGShyy74RNLtzlChEBHJQn3P6oHFHlts+3SpUIiIZKFxRf3o2SN6W2zPHqd3W2wzdWaLiGShVNwW20yFQkQkSxUXFpxRgWimpicREUlKhUJERJJSoRARkaRUKEREJCkVChGRLFNZHeHetXuorI6k5Pl015OISBZJxUJFbemKQkQki6RioaK2VChERLJIQe98cszI4fQXKmpLhUJEJEtUVkeY/8x2mtzJyTHu/MJlKRlwpz4KEZEs8dTmWk7UN+GA4USO1aXkeXVFISKSBSqrI/xy0zt4bDs3NzXNTqBCISKSFTZU7aehKVomDJhZPDglzU6gQiEikhXGFfUjP+/jacVnXD44Zc+tPgoRkSyQymnF21KhEBHJEqmaVrwtNT2JiEhSKhQiIpKUCoWIiCSlQiEiIkkFWijMbJKZ7TKzPWZWFuf4V83sQzPbEvu4OYicIiLdWWB3PZlZLnAvMBGoBTaa2Sp339Hm1Mfd/ba0BxQRESDYK4qxwB53r3L3OmA5MDXAPCIiEoe5+6nP6opPbDYTmOTuN8e2bwRKWl49mNlXgR8BHwJvAX/n7u8keL65wFyAAQMGFC9fvrzV8SNHjtCnT58u+ErOnLJ1XlhzgbKdrrBmC2suSG22CRMmVLr7mLgH3T2QD2Am8GCL7RuBe9qc0w/oGXv8deCVjjx3cXGxt7V27dp2+8JC2TovrLncle10hTVbWHO5uz/w1Et+zyu7fdPeA2f8XMAmT/A3NciR2e8Cn2qxPTi27yR3b7k004PAP6Uhl4hI6FVWR/injcdp8F0pW/I0kSD7KDYCw8zsQjPLB2YBq1qeYGYXtNi8HtiZxnwiIqG1oWo/9U2kdMnTRAK7onD3BjO7DVgD5AIPu/t2M5tP9BJoFfC/zOx6oAE4AHw1qLwiImEyrqgfPXKg0VO35GkigU4K6O6rgdVt9t3Z4vF3ge+mO5eISNgVFxbwD1f04sS5hSmfLbYtzR4rIpKhLirIpbT0oi7/PJrCQ0REklKhEBGRpFQoREQkKRUKERFJSoVCRESSUqEQEZGkVChERCQpFQoREUlKhUJERJJSoRARyRCV1RHuXbuHyupIWj+vpvAQEckAldURbrj/NeobnR65xmNzr0zb59YVhYhIBljyq7epa3QcqGt0ntxcm7bPrUIhIhJyldURXvntH1rtszR+fhUKEZGQ21C1v3l5aAByc4wvXj44bZ9fhUJEJOTGFfUjPy+HHIO8HOMHU0d06foTbakzW0Qk5IoLC3j05nFsqNrf5YsUxaNCISKSAYoLC9JeIJqp6UlERJJSoRARkaRUKEREJCkVChERSUqFQkREklKhEBEJqaAmAWxLt8eKiIRQZXWELz+4gbqGJvLzcnj05nG6PVZERD721OZaTtQ30eRQ39DEhqr9gWVRoRARCZnK6gi/3PQOzbM75ebmMK6oX2B5VChEREJmQ9V+6hujZcKAmcWDA2t2AhUKEZHQKeidf/JqwoERg84JMo4KhYhI2Gx/79DJxzkGkWN1AaYJuFCY2SQz22Vme8ysLM7xnmb2eOx4hZkNDSCmiEjaVFZHeHzTOye38wLun4AAC4WZ5QL3ApOBS4EbzOzSNqfNASLufhHwb8CP05tSRCS9ntpcS0Pjx4sUlV58fqD9ExDsOIqxwB53rwIws+XAVGBHi3OmAnfFHj8B3GNm5i2XekqxyuoIT22uPdkuuO29Q+w7fILz+/bki5cH26EkItmv7R+3/n17BpKjJevCv7nJP7HZTGCSu98c274RKHH321qcsy12Tm1s++3YOfviPN9cYC7AgAEDipcvX97q+JEjR+jTp0/STHsijSzceJyGpvjH83Lgr/44nyP1zh+fl8tFBbkd/nqT6Ui2oIQ1W1hzgbKdrrBmS3euPZFGFr5+nEaHXIOysb0S/q1JZbYJEyZUuvuYeMeyZmS2u98P3A8wZswYLy0tbXV83bp1tN3X1va1e2hs2pXweGMT/Py39TS5k5/X2GqkZGV15LRXn+pItqCENVtYc4Gyna6wZkt3rlJg9OUd+3uSrmxBFop3gU+12B4c2xfvnFozywPOAbpseOK4on70yMuhLsElRU6O0djkOB+PlCwuLGg11D7HjJuvvpBfvfUh70SOce3wASyaNbqrIotIFgpyNbt4giwUG4FhZnYh0YIwC5jd5pxVwFeA14CZwCtd2T9RXFjAY18bl7CP4rJB5zD/me3UNzTRI+/jOxE2VO2nriE61L7JnSXlVSefc+WW99j1wWFuvHIokWN1gax3KyLhdyatEl0tsELh7g1mdhuwBsgFHnb37WY2H9jk7quAh4Cfmdke4ADRYtKlTlXJLxnYt90Pc1xRP3LMaEpQw3Z+cJh5K7ZiQM8eOdz5hctUNETkpDBNABhPoH0U7r4aWN1m350tHh8HvpTuXMnEKyTFhQXMnzqCO5/edrJpKh4H6hqauPPpbTS5k5djfGnMp7iQRkq7OriIhFbLVomWzdphkbBQmNm9wDJ3/+805slYs0uGnLzaKOidz89e28ve/Uc5EevvaPLooJUc+7ifo67RebSihlyD3oNqmF0yJNCvQUSCMa6oH/l5Oe2atcMi2RXFW8C/mNkFwC+Ax9z9jfTEykwtrzaa/+g3tzsW9M4ncqyOgt75zH9mOyfqm05eeTQ63Pn0Ni4Z2BcgtO2UItI1igsLePTmcaF97ScsFO7+78C/m1kh0b6Bh83sLOAxokXjrTRlzGjxmqouGdiXJzfX8vjGd2hsipaLJnee2lzLk7E56M1g7p8WUTZleBCxRSTNwnanU0unnMLD3avd/cfuPhq4AZgG7OzqYNmsuLCABdNH8oOpI8jLMQzIz8vB4eSVRpPDkvIqPvvjV5i7dFPgSyGKSPd1ys7s2PiFyUSvKq4B1vHxtBpyBpr7NR57aSM3XHsFAMtfr6HlzVPvRj7i3chHvLTz9/TKyyEvN4fZY4foSkMkS4T5tthmyTqzJxK9gpgCvA4sB+a6+9E0ZesWigsLOPzp/JO/IHP/tKjVOIxmTQ7H6pugvokl5VU88UYt37r2EnWAi2SwyuoIN9z/GvWNTo9c47G5V4ayWCRrevousB4Y7u7Xu/syFYmuVzZlOAumj+Si889Oet6+w3XMW7GVzy58WU1TIhnqyc211DV+fBfkk5trg44UV7LO7D9PZxD52OySIcwuGcKyihoe31jDoY/q2bv/WNxz3z14nHcPHueFHb/niqEFlE0eHsp3JCLS3p7fH261ve/wiYCSJKcV7kJsdskQnr7tatb9nwncMr6I3vnJf1wb90b40uL1uroQyQCV1ZF2r9UwTCkeT9bMHpvtyqYMp2zKcCqrIyz51dvseP//8W7ko3bnNQFf/9kmPnnuWfzlFUPUhyESUhuq9reaxSHXYMblgwPLk4yuKDJMcWEBD9w0hv/+zp9zy/iiuOfsO1LHm7WHmLdiK5fd+TzLKmrSnFJETqWgdz45Fr09Pi/H+MG0kaFtNlahyGBlU4bz5K1XMTw2ojueo3WNzFuxldJ/XqsmKZGQqKyOMP+Z7TQ2Obk5xvypI0J99a9CkeGKCwt47pvjWTB9JJ8ZfE7CH+je/ceYsXg90+55Na35RKS95kkAHXB3Isfqgo6UlApFlmju+P7lrVcxMEmH2JbaQ1w+/wVdXYgEqLnZKccI5SSAbalQZJniwgI23H4t00YNIi/H4p5z4Fg9X1qynqn3vKr+C5E0a9nslGPGnV+4LLR9E81UKLLUolmj2bNgCtNGDYp7vMk52eF900MVaU4n0n1lWrMTqFBkvUWzRvPkrVfR7+weCc8p372Pkd97Xs1RImnQvPZEboY0O4HGUXQLxYUFVP7j506O9H7v4Ed8eKT1u5jDJxqZsXg9t4zX1OYiXaV5AsBMWw5ZhaIbaZ4apLI6wszF6+Mu2bqkvIrNNRG+o6lARFKqsjrCDQ9sOLmK3WNfC9e62Mmo6akbKi4s4Ilbr6Jvz9y4x1/fG53RUk1RIqnz1Obak30TdQ1NPBXSCQDjUaHopooLC9j6/UmMH9af3Dh3R9U3OvP/azuPbDuhgiGSAm2v4ONd0YeVCkU3t3ROCW8vmMIt44uwFvXCid4Vtba2gRse2KBiIXKGRgw6h1wjuqJlroV2Xqd41EchQHQ6kImXDeSpzbVse/cQb9YeOnmsvqGJ+f+1ncs+eQ4zLh+cMe2qImHRPHaiySE3x7jr+hEZ9TpSoZCTmhd3b151q64xenHcfHXxZu0hllXUcH6ffP5uolbXE+moTBw70ZKanqSd4sICHpt7JbNLhlD0ifa/Ih8eia6uN/En69IfTiQDZeLYiZZUKCSu4sICFkwfyezh+eTnxf812f3hUa5e+HKak4lknuLCAh69eRzf+twlPHpz5twW20yFQpK6qCCXx742LuFU5rUHj2sKc5EkllXUcONDFez64DDfmHBRxhUJUKGQDmieyvyW8UXEm2Zw7/5jzFy8noWrd6Y9m0iYLauoYd6Krfx69z7mrdiasZNwqlBIh5VNGc4Tt14V95gTHdWtCQZFPvbctveTbmcKFQrplOLCAp689SoKz+sd93j57n388R3P6epCBJg84oKk25kikEJhZueZ2Ytmtjv2b9xGOzNrNLMtsY9V6c4p8RUXFvCrf5iQcM3u4w1NLCmvUke3dGuV1REix+q4ZXwRfzqsf/TmkAy9pTyoK4oy4GV3Hwa8HNuO5yN3HxX7uD598aQjmtfsTnR1UXvwuFbTk25pWUUNf3nfa/zLml389LW9fPPaizO2SEBwhWIq8Ejs8SPAtIByyBlqvroYP6x/3OMHjtUzY/H6jO3EE+msyuoI//j0NhqaPDoBYH0TG6r2Bx3rjJh7+qemMrOD7n5u7LEBkebtNuc1AFuABmChu69M8pxzgbkAAwYMKF6+fHmr40eOHKFPnz6p+QJSLFuyraup5+c762hI8CtV1Ne487Pxrz66Mle6KdvpCWu2zuZ6ZNsJ1tY2nNzOAeaV9OKigvizNaczWzITJkyodPcx8Y51WaEws5eAgXEO3Q480rIwmFnE3dv1U5jZJ939XTMrAl4BrnH3t0/1uceMGeObNm1qtW/dunWUlpZ27otIk2zLNvEn69j94dG4x/r17sH9X7nijO8lz7bvWbooW+d1NtfcpZt4YcfvT26PHVrAL26Jf7fgmUrl98zMEhaKLmt6cvdr3X1EnI+ngd+b2QWxcBcAf0jwHO/G/q0C1gGjuyqvpM6L3y5N2BS1/1g9s7TWhWSx/n17ttq+aED8waqZJKg+ilXAV2KPvwI83fYEMysws56xx/2BzwI70pZQzsjSOSXcMr6I/Nz4a118bekm9VtIVppx+WDycy0jpxNPJKhCsRCYaGa7gWtj25jZGDN7MHbOcGCTmb0JrCXaR6FCkUHKpgznrR9OYdj5Z7c7duBodGLByYvKdXUhWWNZRQ2LXnqLv/nshfz95y/hsblXZuSUHW0FUijcfb+7X+Puw2JNVAdi+ze5+82xx+vdfaS7fyb270NBZJUz9+K3S1kwfSR94iy9uvODw8zUXVGSBRau3nlyuo4l5VUU9M7PiiIBGpktaTK7ZAjzplwa95gDt2fwPDgildUR7iuvarUvU6friEeFQtJmdskQFkwfSf+++e2OOTBvxVam3fNq+oOJnKEnN9e2WwM7U6friEeFQtJqdskQNt0+MeH0H1tqD2nqD8k4bW/ZGDu0IKNHYrelQiGBKJsynAXTR8b9Baw9eFyz0ErGqKyO8OHhE+RYtGDk5+XwncnDg46VUioUEpjZJUP45a1XMbDNfecQnYX2irtfVL+FhFpldYSZi9fzwo7f0+SQY3DX/7gsazqxm6lQSKCKCwvYcPu1DD63V7tjzWtzq99Cwupbj29p1TfR6BA5VhdYnq6iQiGh8GrZNQlHc2+pPcSo769JcyKR5CqrI1QfONZu/7iifgGk6VoqFBIaS+eUMG3UoLjHDn7UQLGmLJcQ+ceVW9vtGz+sf9Y1O4EKhYTMolmjE94Rtf9YvdbmllCorI6w4/3DrfblWPTNTjZSoZDQaV4QqWeceaKa1+b+xW9PpD+YSEy89SX69MwLIEl6qFBIKBUXFrArwTxRAKv3NjDuhy+pKUoCUdA7v93Yidljs2fcRFsqFBJqzfNExfPB4RPMUFOUpFlldYT5z2zHYuMmzju7B7eML6JsSnaNnWhJhUJCr3nqj/YNUVFLyqv45vI30ppJuq+nNtdyor7p5LiJOVdnd5EAFQrJELNLhvBEgsF5ACu3vMfEn6xLbyjpdhau3smyipqTYydyc3Oy8nbYtlQoJGM0D85LNN5i94dHGXHn82lOJd3FsooalpRXtRpgV3rx+Vl5O2xbKhSScZbOKeHKgfEXqj9S16hJBaVLxJs2vO2yp9lKhUIy0tdH9Up4ZVF78Lg6uCXl2k4bnptDVixz2hHZe+OvZL2lc0qorI4w6/711De2PrakvIol5VUMO/9sXvx2aSD5JHssXL2T57d/wPhh/Tn0UT0DPtGLr//Zp7tFsxPoikIyXHFhAbt/eF3cSQUh2m9R9N1nNQutnLb7thxnSXkVe/cfo3z3Pq4s6sf9N43pNkUCVCgkS7xadk3CqT+aPLp6npqjpLMqqyO89kHry9Xnt38QUJrgqFBI1iibMjxhsYBoc5SKhXTG1x7Z2G7fpMsGBpAkWCoUklU6Uiy0voV0xE0PVXDgWH2rfX3yc7N+cF08KhSSdZonFUw0OG9L7SGuuPvFNKeSTPP63gPt9g0896wAkgRPhUKyUvPgvFGDz4l7/MMjdQy/47k0p5JM0iuv/Z/Hv/nshQEkCZ4KhWS1lbddzbRRg8iJM1HURw1NDC17Vv0W0s60e17l4EcNJ7dzc2DB9JHMLsneGWKTUaGQrLdo1miqfnQd5/fJj3t8SXkVNz1UkeZUElaV1RG21B5qta9Hbk63LRKgQiHdyMY7JnJWnOYEgPLd+9TJLQDcsaL9Eqdjh54XQJLwUKGQbmXn3ZPpkx9/nqgttYd0ZdHNTbvnVXZ+0HqJ04L87F3itKNUKKTb2TZ/UsKR3OW793HJHc+p36Ib+ubyN9o1OQF8Y3T835XuRIVCuqVXy65JOKngiYYmlpRXaRbabqSyOsLKLe+12z9t1CAuKoh/BdqdBFIozOxLZrbdzJrMbEyS8yaZ2S4z22NmZenMKNlv6ZwSnrz1KgrP6x33eO3B41oMqZv48XPtryBHDT6HRbNGB5AmfIK6otgGfBEoT3SCmeUC9wKTgUuBG8zs0vTEk+6iuLCAX/3DhKSLIY383vNUVkfSnEzS5ZvL3+D1va1/vsMH9mXlbVcHlCh8AikU7r7T3Xed4rSxwB53r3L3OmA5MLXr00l3tHROCdNGDYp77PCJRmYsXq91ubPQsoqadk1OBtw9fWQwgULK3P3UZ3XVJzdbB/y9u2+Kc2wmMMndb45t3wiUuPttCZ5rLjAXYMCAAcXLly9vdfzIkSP06dMntV9Aiihb53VVrnU19fx0R13C4+f0gH+/5uykzxHW7xkoW1v/+5WjHGrz4+7fC/6l9OOfcXf5nk2YMKHS3eN2BXTZwkVm9hIQb5rF29396VR/Pne/H7gfYMyYMV5aWtrq+Lp162i7LyyUrfO6KlcpcHFFDXet2kZdY/s3UYfqYc6ao/zilqsSrkcQ1u8ZKFtLyypqOFTXfszEtyaPpLTF4Dp9z7qw6cndr3X3EXE+Olok3gU+1WJ7cGyfSJeaXTKEt344JeFI7kaHGYvX6xbaDPevL7Vv/Z42alC3HoGdSJhvj90IDDOzC80sH5gFrAo4k3QjG++YmLCTG6JTf6jfIjMtXL2TfYdbtzn175Ovu5wSCOr22OlmVgtcCTxrZmti+weZ2WoAd28AbgPWADuBX7j79iDySve1dE4JC6aPJM6cggCs3PKexltkmGUVNSwpr2q3/1sTLwkgTWYI6q6nFe4+2N17uvsAd/98bP977j6lxXmr3f1id/+0u/8wiKwis0uG8LuFiScVrD14XOtbZIhlFTXMizOXk5qckgtz05NIqGy8Y2LS9S0+8/01LKuoSXMq6ahEReKKoQVqcjoFFQqRTlh529UJ+y0OfdTAvBVbmbPmqPouQqayOsLtcYoEQNnk7re0aWepUIh0UnO/RYIZy2n0aN+Fpi0Pj2/8vJJ4I8amjRqU8DZn+ZgKhchpmF0yhD0Lrkt6V9SW2kOaKyoErl74Mh8cPtFu/7RRg9Tk1EEqFCJnoPnqIlFHt+aKCtao76+h9uDxdvtvGV+kItEJKhQiZ2h2yZCkHd3Nc0VpUaT0GvX9Na3WvT65f/A5lE1Rv0RnqFCIpMjK265m2qhBCcdclO/ex0XzVmtEdxrc9FBF3CIx+NxemhX2NKhQiKTQolmj+c9JZydcQa+hyVlSXqWriy407Z5XKd+9r93+XIsuWCWdp0Ih0gWSraAH0auLv1iyXn0XKXb1wpfjLmcK8INpmjr8dKlQiHSR5o7us3rEf5m9vjfCzMXrNUgvRRau3hm34xqi/RIaeX36VChEutDskiHs/MFkbhlfRK84Ay8cmLdiK1fc/aIKxhlYVlHDg6/+Lu6xUYPPUb/EGVKhEEmDsinD+e3d0YIRz4dH6pi3YisXlj2rzu5OWrh6J/NWbKWhqf2QummjBqlIpIAKhUgalU0ZnnDJVYheYSwpr9KMtB1UWR2JOxMsaKxEKnXZCnciEt+iWaMZ+IleLN1QzbG6xrjn1B48ztCyZxk/rD9L55SkOWH4VVZH+PFzO3njnYNxjy+YPlJ9EimkKwqRAJRNGc6O+ZOSXl1A9O4ojexubeHqncxYvJ7X90aoj7NcrYpE6qlQiARo0azRPHnrVeQmGqXHxyO7L5//QrcvGAtX70zY1ATR5iYVidRToRAJWHFhAW//6DqGnX920vMOHKtnxuL13XYK82n3vHrKIqGpObqG+ihEQuLFb5eyrKKGf3txFx8eqUt43sot77Fyy3vdpv9i4eqd3FdeFXeacIDzevfgga9coenCu5AKhUiIzC4ZwuySIVRWR/jyAxs43tCU8Nzy3fsYWvYsY4cW8J3Jw7PyD+WIO5/nSIIOf4Dz++Sz8Y6JaUzUPanpSSSEigsL+O3dk0/ZHAXREd4zFq/n0999NmsG7VVWR7iw7NmkRWL8sP4qEmmiKwqREHvx26UAfHP5G/zXm+8R5yafkxo9Osr7zqe38YU/uSAjxxAsq6jhrlXbqEvyhZ57Vh5bvvf5NKYSXVGIZIBFs0bz9o+uSziyu6WGJmfllvcYWvYskxeVZ8SdUssqapj7wlHmrdiatEic3ydfRSIAuqIQySBlU4ZTNmU4E3+yjt0fHj3l+Ts/OMyMxesx4IfTR5J81Eb6dfTrAC1dGiQVCpEM1LJJauWW9055fvPkgwB9XnmeedddGth4g2UVNSx4dkfS/oeW1GEdPBUKkQy2aNZoFs0a3al35kfqGpm3Yivfe3orPfNyaXTn85cN7NJ36zc9VMGvd+9LeItrPLkWXUNCA+iCp0IhkgWarzCg48059U1QH3tX3zw2w+DkH/PT7TSurI7wjZ9X8sHhE53+vwAGfF2D50JFhUIkyzQXjWUVNdyxYiuJR2K01/Id/8GPGhha9mwqo51SdxlEmGlUKESyVPPgPYAr7n4x6WjvIDWPrD78uzcpLVWRCCMVCpFuoLkzeN26dTz89lmsf3s/7p50XEZX6pFjXNdmrMe6+AvUSQioUIh0My2bdhau3skDv66i0aOdx11ZOD7RK4///OuxWTnVSLYLpFCY2ZeAu4DhwFh335TgvL3AYaARaHD3MenKKNIdNI/LaKmjt9wmk5cD86fqjqVsEdQVxTbgi8B9HTh3grvv6+I8IhLTfMutSLNACoW77wQwS7Jai4iIhELY+ygceMHMHLjP3e9PdKKZzQXmxjaPmNmuNqf0B8J6ZaJsnRfWXKBspyus2cKaC1KbrTDRgS4rFGb2EjAwzqHb3f3pDj7N1e7+rpn9EfCimf3W3cvjnRgrIskKyaaw9nEoW+eFNRco2+kKa7aw5oL0ZeuyQuHu16bgOd6N/fsHM1sBjAXiFgoREekaoZ1m3MzONrO+zY+BzxHtBBcRkTQKpFCY2XQzqwWuBJ41szWx/YPMbHXstAHAq2b2JvA68Ky7P38GnzZhs1QIKFvnhTUXKNvpCmu2sOaCNGUz94CGZoqISEYIbdOTiIiEgwqFiIgklXWFwswmmdkuM9tjZmVxjvc0s8djxyvMbGiIsn3LzHaY2W/M7GUzS3hfc7qztThvhpm5maXldsGO5DKzv4h937ab2bJ05OpINjMbYmZrzeyN2M90SppyPWxmfzCzuDd/WNR/xHL/xswuT0euDmb7cizTVjNbb2afCUu2FuddYWYNZjYzLLnMrNTMtsReA79KeQh3z5oPIBd4GygC8oE3gUvbnPO3wJLY41nA4yHKNgHoHXt8a5iyxc7rS/T25A3AmDDkAoYBbwAFse0/Csv3jGhH462xx5cCe9OUbTxwObAtwfEpwHNE1wgaB1SkI1cHs13V4mc5OUzZWvzcXwFWAzPDkAs4F9gBDIltp/w1kG1XFGOBPe5e5e51wHJgaptzpgKPxB4/AVxj6ZlL5JTZ3H2tux+LbW4ABqchV4eyxfwA+DFwPES5vgbc6+4RiI65CVE2Bz4Re3wOcGYz7XWQRwelHkhyylRgqUdtAM41swvCkM3d1zf/LEnva6Aj3zeA/wk8CaTr96wjuWYDT7l7Tez8lGfLtkLxSeCdFtu1sX1xz3H3BuAQ0C8k2VqaQ/RdXzqcMluseeJT7p7OJc868j27GLjYzP7bzDaY2aQQZbsL+KvYreCrif6RCYPO/i4GJZ2vgVMys08C04HFQWdp42KgwMzWmVmlmd2U6k8Q9rmeuiUz+ytgDPBnQWcBMLMc4F+BrwYcJZ48os1PpUTffZab2Uh3PxhkqJgbgJ+6+0/M7ErgZ2Y2wt07szppt2RmE4gWiquDztLCIuA77t4UsglN84Bi4BrgLOA1M9vg7m+l8hNkk3eBT7XYHhzbF++cWjPLI9oksD8k2TCza4HbgT9z99NbnT712foCI4B1sRfIQGCVmV3vCdYSSVMuiL4brnD3euB3ZvYW0cKxsQtzdTTbHGASgLu/Zma9iE7ilrZmiwQ69LsYFDP7E+BBYLK7p+O12VFjgOWx10B/YIqZNbj7ykBTRV8D+939KHDUzMqBzwApKxTZ1vS0ERhmZheaWT7RzupVbc5ZBXwl9ngm8IrHeoCCzmZmo4mu0XF9GtvaT5nN3Q+5e393H+ruQ4m2HXd1kThlrpiVRK8mMLP+RC/Dq7o4V0ez1RB9l4eZDQd6AR+mIduprAJuit39NA445O7vBx0KoneKAU8BN6byHXEquPuFLV4DTwB/G4IiAfA0cLWZ5ZlZb6AE2JnKT5BVVxTu3mBmtwFriN6d8LC7bzez+cAmd18FPES0CWAP0Q6iWSHK9s9AH+CXsXctNe5+fUiypV0Hc60BPmdmO4iuhPh/0vEutIPZvg08YGZ/R7Rj+6vpeFNiZo8RLZ79Y/0j3wN6xHIvIdpfMgXYAxwD/rqrM3Ui251E+wz/b+w1kLaVLTuQLRCnyuXuO83seeA3QBPwoLundF48TeEhIiJJZVvTk4iIpJgKhYiIJKVCISIiSalQiIhIUioUIiKSlAqFiIgkpUIhEoDYYDe9/iQj6BdVJE3MbGhs/YqlwDZaT6MhEloacCeSJhZdJKsKuCo2vbdIRtAVhUh6VatISKZRoRBJr6NBBxDpLBUKERFJSoVCRESSUme2iIgkpSsKERFJSoVCRESSUqEQEZGkVChERCQpFQoREUlKhUJERJJSoRARkaT+P90JAcZym0QTAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAlCElEQVR4nO3de3zU9Z3v8dcnCQExVFOxoCIgBVsotGKooZSDcLxUcFelaItsa9vVpbjlnHXr7p4UrXatpXTP7jl0H7qgq261pzS6WpBHRa0XkKVIhCgtClUwNWlEETAgFzG3z/ljZmCSzEwSmPx+v5m8n49HHs7v4syHJJP3/L63n7k7IiIi6RSEXYCIiESbgkJERDJSUIiISEYKChERyUhBISIiGRWFXUBPGDhwoA8fPrzNvkOHDnHyySeHU1AnVFv3RbUuUG3HK6q1RbUuyG5t1dXVe9z99JQH3T3vvsrKyry91atXd9gXFaqt+6Jal7tqO15RrS2qdblntzZgk6f5m6qmJxERyUhBISIiGSkoREQkIwWFiIhkpKAQEZGMFBQiIpKRgkJERDJSUIiISEYKChERyUhBISIiGSkoREQkIwWFiIhkpKAQEZGMFBQiIpKRgkJERDJSUIiISEYKChERyUhBISIiGYUaFGb2gJm9Z2avpjluZvavZrbDzH5vZucHXaOISG8X9hXFz4DLMhyfDoyKf80FlgRQk4iIJAk1KNx9LfB+hlOuBB6K3/t7A3CqmZ0RTHUiIgLhX1F05izgT0nb9fF9IiISEHP3cAswGw782t3Hpjj2a2CRu6+Lbz8H/C9335Ti3LnEmqcYNGhQWWVlZZvjBw8epKSkJPv/gCxQbd0X1bpAtR2vqNYW1bogu7VNmzat2t0npDzo7qF+AcOBV9Mcuwe4Nmn7deCMzp6zrKzM21u9enWHfVGh2rovqnW5q7bjFdXaolqXe3ZrAzZ5mr+pUW96WglcFx/9NBHY7+7vhF2UiEhvUhTmi5vZL4GpwEAzqwduB/oAuPtSYBUwA9gBHAa+FU6lIiK9V6hB4e7XdnLcge8EVI6IiKQQ9aYnEREJmYJCREQyUlCIiEhGCgoREclIQSEiIhkpKEREJCMFhYiIZKSgEBGRjBQUIiKSkYJCREQyUlCIiEhGCgoREclIQSEiIhkpKEREJCMFhYiIZKSgEBGRjBQUIiKSkYJCREQyUlCIiOSR6toG7l69g+rahqw9Z6j3zBYRkeyprm3gL+7bQGNzK8VFBfzihomUDSs94efVFYWISJ7YULOXxuZWWh2amlvZULM3K8+roBARyRMTR5xGcVEBhQZ9igqYOOK0rDyvmp5ERPJE2bBSfnHDRDbU7GXiiNOy0uwECgoRkbxSNqw0awGRoKYnERHJSEEhIiIZKShERPJAT8yfSFAfhYhIjuup+RMJuqIQEclxPTV/IkFBISKS43pq/kRCqEFhZpeZ2etmtsPMKlIc/6aZ7TazzfGvG8KoU0QkyhLzJ7576aey3uwEIfZRmFkhcDdwCVAPbDSzle6+td2pD7v7/MALFBHJIT0xfyIhzCuKC4Ad7l7j7o1AJXBliPWIiEgK5u7hvLDZ1cBl7n5DfPvrQHny1YOZfRP4MbAbeAP4W3f/U5rnmwvMBRg0aFBZZWVlm+MHDx6kpKSkB/4lJ061dV9U6wLVdryiWltU64Ls1jZt2rRqd5+Q8qC7h/IFXA3cl7T9deCuduecBvSNP/428HxXnrusrMzbW716dYd9UaHaui+qdbmrtuMV1dqiWpd7dmsDNnmav6lhNj29DZydtD0kvu8od9/r7h/FN+8DygKqTURE4sIMio3AKDM7x8yKgdnAyuQTzOyMpM0rgG0B1iciEmk7Glp6bDZ2stBGPbl7s5nNB54GCoEH3P01M7uD2CXQSuB/mtkVQDPwPvDNsOoVEYmSZVV1/PilI7T66/Ttk/3Z2MlCXcLD3VcBq9rtuy3p8feA7wVdl4hIlFXXNnDb46/SEh+L1NgUm42dj8NjRUTkOGyo2Utr0ojVggLL+mzsZAoKEZEck1iyw4CiAuOOK8f22NUEaPVYEZGc9OXzh/DO2zuZ/+cX9GhIgIJCRCSnJC8pXmQQxPpGanoSEckhbZYUbyXrS4qnoqAQEckhpf2LaY33Y3t8u6cpKEREckjD4UYs/tji2z1NQSEikkNK+xdTWGAUAH0K6NFhsQkKChGRHLGsqi420a7VKSgw5ny6uMdHPIFGPYmI5ITEbOzmeAdFqzsHm4K5TYSuKEREcsCGmr20tCbNxjbj0x8vDOS1FRQiIjlg4ojT6NungAKOzcYeWRpMUKjpSUQkB5QNK+UXN0xkQ81eJo44jbJhpaxZUxPIaysoRERyRNmw0kA6r9tT05OIiGSkoBARkYwUFCIikpGCQkREMlJQiIhIRgoKEZEIq65t4O7VO6iubQitBg2PFRGJqOSbFBUXFfCLGyZqeKyIiBzz2Mv1fNQUv0lRc2sgNylKRUEhIhJB1bUNPFpdT2J1p8ICC2RJ8VQUFCIiEbShZi/NLa1A7AZF10w4O5RmJ1BQiIhE0sQRp1FcVEChQd8+BXz5/CGh1aLObBGRiJp1/hA8/t+wriZAQSEiEjntRzvNCvFqAtT0JCISORtq9tLYHP5opwQFhYhIxCT3T/QpKghttFOCmp5ERCIm1U2KwhRqUJjZZcBPgULgPndf1O54X+AhoAzYC3zV3d8Kuk4RkaCFdZOiVEJrejKzQuBuYDowBrjWzMa0O+16oMHdRwL/F/hJsFWKiAQrCms7tRfmFcUFwA53rwEws0rgSmBr0jlXAj+IP34UuMvMzN2dHlZd28CvXq6PxNA0EekdorK2U3sWwN/c1C9sdjVwmbvfEN/+OlDu7vOTznk1fk59fPvN+Dl7UjzfXGAuwKBBg8oqKyvbHD948CAlJSVdqm1HQwuLNh6hOTYpkkKDKWcV8sWz+jCytJAdDS384f0WPv3xQkaWFnb7395ed2oLWlRri2pdoNqOV1RrC7KuX7/ZyGPbm3BizT1fHtWHP/tkcSC1TZs2rdrdJ6Q6ljed2e5+L3AvwIQJE3zq1Kltjq9Zs4b2+9J5bfUOWlpfP7rd4rCmvoUXdzm3/dln+OfnXosnfsvRxK+ubTjujqfu1Ba0qNYW1bpAtR2vqNYWZF0Dzmng129toKm5lT5FBVx78ecz/j0JqrYwg+Jt4Oyk7SHxfanOqTezIuAUYp3aPWriiNPoU1RAY+KSAnBi45mffPWdlOObo3i5KCK5JWqjnRLCnEexERhlZueYWTEwG1jZ7pyVwDfij68Gng+if6JsWCm//KuJ/EX5UC4dM4jiQjs6nnn62DM6jG9ONznmpspXOO+O33BT5Ss9XbKI5ImyYaV8Z9rIyIQEhHhF4e7NZjYfeJrY8NgH3P01M7sD2OTuK4H7gZ+b2Q7gfWJhEojkoWntm5U+NXhAh8QvLio4erk4ccRp3FT5Cis27wQ4+t/Fs8cHVb6I5JgTab7uaaH2Ubj7KmBVu323JT0+AlwTdF3ttR/PnGq7/eXi9Q9ubPMca97YffRx4hdi+64DbP7TPsYMaCSCTbMiEpCojnZKSBsUZnY3sMzdfxtgPTmrfXhMPff0o1cSiW049gtxpOlY/8dbe+GapeupmD46Ur8cIhKM5ObrxnjzdZT+FmS6ongD+GczOwN4BPilu6uxvYsSzUxr3tjN1HNPP7qd+IVob+NbDXz1nvUM/lg/+vUp5C8nj2BO+dBAaxaRcGzfdYDWeO9rq0Np//RDYsOQNijc/afAT81sGLG+gQfM7CTgl8RC442AasxZqfokEot9JV9RJDS3Qv2+IwAsWL6Fur2HqJgxusfrFJHwLKuqa9P6ANBwuDGkalLrtI/C3WuJLZ3xEzMbDzwA3EasA1q6Kbk/Y/uuA2yo2ct7H3xEx9iAe/+rBoAXa/Yy6GP9+PaFn4zU5aiInLiHN9a12TYIfbXY9joNivj8henEriouAtZwbFkNOQ7t+zPuW/4cVQdO4Zmtu9qc1+qwdG1NfGs/z27bxZ1XjVOTlEgead8U/enBAyL3gTBTZ/YlwLXADOAloBKY6+6HAqqt1xhZWsgNMyewrKqOf1u9nfp9RzDAjKPtlhB7fNvjr/LIxjr+sOsAFwz/OA9dXx5a3SJyYqprG/jDuwfa7BsfsZCAzFcU3wOWATe7e3SWMcxjc8qHMqd86NHhswc+bEq6oohpbnU21+8HYO32PUz80bPc/bWyyH0CEZHO3fPCmyTPIDYI/banqWTqzP7vQRYixyQ3TQ097WQeWFdDzZ7YhVxru3np7x74iFlL1vOxfkV8avAADbEVySGJ93XCJz9REsn3r26FGnFzyofy7M1T+c95k7j50k9x3pBTUp73wZFmNr7VwNVL1kdqHXsRSa26toG39rYNir/84jkhVZOZgiJHJNZ/WTF/ctqwgNjihd/6j5f4q4c2KTBEImxDzV5ak5oILh0zKLIDVfJmmfHeZMX8yVTXNnDPC2/ym3YjpSB2dfHM1l08s3UXxYUFzBg3WOtMiURMYk5VYo24b1/4ybBLSktBkaPKhpVy73UTqK5t4LsPb6b2/cMpz2tsaWXF5p28tecQK+ZPDrhKEUknqkuKp6KmpxxXNqyUF/5hGgtnjmPk6SenPW9z/X5m/HStmqNEIiBxX2wgckuKp6IrijyRGFq7aNW2DkNqE7a+c4BZS9bTv7iQ6yYO0/IgIiGI+kqxqeiKIs9UzBjNYzdO4tIxgzipT+of7+HGFpaureHCf1qtKwyRgD32cj0fNXW80VmUKSjyUKL/YtsPp3PVeWemPa/2/cNcvWQ9y6rq0p4jItlTXdvAo9X1RyfZFRZY5NZ1SkVBkecWzx7PwpnjGDgg9bLFTmyl2vN/+AyLVm0LtjiRXmZDzV6aW2JrOxlwzYSzI9/sBAqKXmFO+VA23XIJ86aMoG9R6h/5+4caWbq2huvurwq4OpHe48CHTUdXV3DgM2emnxMVJQqKXqRixmhevzNzc9Ta7Xv4ylLN7hbJturaBu5b98ej20b07juRjoKiF0o0R6UbTvvSWw3MvvdFhYVIFm2o2UtL0kzsXOmfAAVFr5VYQ2rhzHEpjze1ON/6j5dYsPaQOrtFsmD7rmPLiRcVGHdcOTYn+idAQdHrzSkfymM3TuKC4R1/YT840szOw7HObvVdiBy/Rau2sWLzzqOjnf7ss2dEdl2nVBQUQtmwUh6ZN4mFM8dRaKnPWbt9DzdVvhJsYSJ54qnX3m2zvflP+8Ip5DgpKOSoOeVDeWTeJC4ZMyjl8RWbd3Lxv6xhrlamFemWyz4zOON21CkopI2yYaX8+3UTWDhzHKee1HGFlx27D/Gbrbu4RiOjRLrsks8M5oLhpQz+WF/mTRmRc8vnKCgkpTnlQ9l8+5f4wuBCUrVGtTrMvvdFTdIT6cSyqjq+es+LbKptYN+HTVySY1cToKCQTnz7vH48Gl87qr2mFmfp2hom/uhZXV2IpFBd28Btj79Kc6vT6tCYI2s7taegkE4l1o6aMmpgyuOJ+3ZrZJRIW+3nThRY7sydSKagkC576PryTmd1X/Iva4IrSCTiSvsXU1hgGLk3dyKZgkK6ZfHs8Tx24yQGD+ib8vj23YeYvlg3SBJZVlV3tNmpMB4SuTR3IlkoQWFmHzezZ8xse/y/KSPWzFrMbHP8a2XQdUpqZcNK2XDLxcybMiJlR/e2dw/w1Xs0Kkp6r+S+CYDWVs+ZdZ1SCeuKogJ4zt1HAc/Ft1P50N3Pi39dEVx50hUVM0bzozRLgDS3wnd+Ua3lP6RX2lCzl1ZP6pvIoXWdUgkrKK4EHow/fhC4KqQ65ATNKR8au99FScf7Xbz7wUcsWL6FUbes0qxu6VUOfNh09HEu900kmCelXmAvarbP3U+NPzagIbHd7rxmYDPQDCxy9xUZnnMuMBdg0KBBZZWVlW2OHzx4kJKSkuz8A7IsX2pbU9fEQ9saaU3zK3Vmf1g4JfWKtT1ZV9BU2/GJam3drWtNXRM/23qsmWnG8CK+8unUfXonKpvfs2nTplW7+4RUx3osKMzsWSDVzJJbgAeTg8HMGty9Q9ya2Vnu/raZjQCeBy5y9zc7e+0JEyb4pk2b2uxbs2YNU6dO7d4/IiD5VFt1bQNLX3iTZ7buSnl8YEkx373kUyfcqZdP37Mgqbbu625dV961jt/V7z+6/d9GDeTn15f3QGXZ/Z6ZWdqg6LGmJ3e/2N3Hpvh6HNhlZmfEizsDeC/Nc7wd/28NsAYY31P1SnYklgBJN4x2z8FGFizfoqYoyUvVtQ28unN/m33Tx54RUjXZE1YfxUrgG/HH3wAeb3+CmZWaWd/444HAF4GtgVUoJ2Tx7PGMSnNjJIgtMKgJepJv7nnhTeK3xAbgguGlOTskNllYQbEIuMTMtgMXx7cxswlmdl/8nNHAJjP7HbCaWB+FgiKHPHPzVKaMGkhBhqXLR9/6pEZGSV6orm3g2W1tm1xHDRoQUjXZ1XF50AC4+17gohT7NwE3xB+vB1KPvZSc8VC8bXZZVR0Llm/pcPzD5lYWLN9C3d5DObeipkiyRU9uazOQo8Dgy+cPCa+gLNLMbAlEYhhtOkvX1qjfQnLWsqo6Nr7VdoLphGGlOT0kNpmCQgKTuO3qx/v3SXlc/RaSqx7e2LH5dGSeNDuBgkICVjaslJdvuzTtSrRaWFByTXVtA1vf+aDNvkKDWXnS7AQKCgnJQ9eXp22K2r77EJ+/85mAKxI5Pr96uZ7mlmOdEyM/UcIj8yblTbMTKCgkRHPKh6adb7H7YCMX/OgZjYiSSKuubeA/N/2JREwUFxXwk1mfzauQAAWFhGzx7PHMmzKCohRjaN87EJucN3nRcyFUJtK5X71cT1P8asKAq8uG5F1IgIJCIqBixmh2LJzB6SkWFgSo33eEc29ZpasLiZTq2gYe3lh39GqiT1FBXvVLJFNQSGRsvPUSrjrvTE7q0/HXsrHFWbB8i0ZFSWQ89nI9zUmzsC889/S8vJoABYVEzOLZ49n2w+kMObVfyuNrt+/RfAuJhB27DrTZ/kSauz7mAwWFRNK6iovShsWKzTv5u9WHAq5I5JhFq7bxUtIEu8KC/JmFnYqCQiJrXcVFnDfklJTH9nyEhtBKKKprG7hnbU2bfWPPPCVvm51AQSERt2L+5LSr0O4+2Mj0xWt1b24J1K3Lt9D+Lj5f/XzurxCbiYJCIu+Zm6cyb8oIClOsQrvt3QPMWrJendwSiGVVdWx7t23fxFmn9suLpcQzUVBITqiYMZo3f3x52iG0WvpDgvDAupoO+74zbVQIlQRLQSE5JTGENpXtuw9x4T+tVlOU9Bxre1l7eklx3l9NgIJCctDi2eP5wuDClMdq3z/MrCXrNTlPsm5ZVR0fNja32fe3l3wqpGqCpaCQnPTt8/qxcOY4+qWYnAewYPkWhYVkTeLGW2/vOwLEluuYN2VEr7iaAAWF5LA55UP5ww+np12yXDO5JVva32/CgQEnpb6vSj5SUEjOe+j68rT9Fmu371FYyAlZU9fElrf3t9nXp9CYOOK0kCoKnoJC8sLi2eMzhoXmW8jxqK5t4MGtjW3uhT3yEyVUzv1CXk+wa09BIXlj8ezxaZuhEvMttE6UdMdPntzWZnJdgZGX95vojIJC8spD15czb8oI+hal/tVesXkn5/3j0wFXJbmourahzXpOAGecelKvCwlQUEgeqpgxmtfvnJ62KWrfh80KC+nUPS+82WHfmDM+FkIl4VNQSN5aPHt82kUF933YzMSFz6rfQlJaVlXHb7buarOv0GDehZ8MqaJwKSgkr62YPzntlcW7H3zErCXrWbRqW8BVSdTd8evX2mz3Ly7gkXmTemWzEygopBdYPHs8C2eOS3t86doarrprXYAVSZRdddc6jjS1ttnXr09Rrw0JUFBILzGnfCiP3TiJwWnuQra5fj+TFz0XcFUSNcuq6thcv7/D/q+U5e9NibpCQSG9RtmwUjbccnHafov6fUe0Am0vl2p12DP7xwZI9GYKCul1VsyfnHa+xfbdhxjxvSc036IXWlZVR+37h9vsKykuZOGU1DfO6k0UFNIrJeZbpNLqsfkW596yKuCqJCyJRf+aWtreu27B5WNCqihaQgkKM7vGzF4zs1Yzm5DhvMvM7HUz22FmFUHWKPmvYsZoHrtxEqf1T724W2OLM/a2pwKuSsLwwG//2GHfyNNP7jWrw3YmrCuKV4EvA2vTnWBmhcDdwHRgDHCtmSneJavKhpVSfdulafstDja26GZIea66toEd7x3ssP8vJ6e+4uyNQgkKd9/m7q93ctoFwA53r3H3RqASuLLnq5PeKNN8i8TNkDTfIj995/9Vd9jXm+410RXm7p2f1VMvbrYG+Dt335Ti2NXAZe5+Q3z760C5u89P81xzgbkAgwYNKqusrGxz/ODBg5SUlGT3H5Alqq37eqquHQ0t/LjqCC1pjp/Zn047N6P6PQPV1t4dvz1MzYG2fwOLC+DeS4/9jHvL92zatGnV7p6yK6AoK6+Qgpk9CwxOcegWd38826/n7vcC9wJMmDDBp06d2ub4mjVraL8vKlRb9/VUXVOBG2bCdfdXsXb7ng7Hdx6Gv1/XxMZbLwm8tmxQbcdcd38VNQcOddg/47NnMnXq+NDq6o6gauuxpid3v9jdx6b46mpIvA2cnbQ9JL5PpMdluhnS7oON6uTOcdW1DSk/CJw35BQWzx6f4v/o3aI8PHYjMMrMzjGzYmA2sDLkmqQXyXQzpIONLQyv0HyLXLXoyY79TYMH9GXF/MkhVBN9YQ2PnWlm9cAXgCfM7On4/jPNbBWAuzcD84GngW3AI+7+WrrnFOkJi2ePZ96UEWnfKCs272R4xRPq6M4hV921jo3t7jNhwN1fKwunoBwQ1qin5e4+xN37uvsgd/9SfP9Od5+RdN4qdz/X3T/p7j8Ko1aRihmjqVl0OSXFhWnPWbq2Rst/5ICr7lrXYS2nkr6FPHpj710Ztiui3PQkEimv3nEZQ07tl/b49t2HtLBghC1atS3lgn9fKx+mkOiEgkKkG9ZVXMRjN04izZ1Wqd93hL95/hDLquqCLUw6teyljj+TwQP69voF/7pCQSHSTWXDStmx8PK0YbG/ERYs36J7XETIolXb+OBIc4f96pfoGgWFyHHasfDyjE1Rm+v3M+qWJ7T8R8iuumsdS9d2XD583pQRanLqIgWFyAlYV3FR2lVoAZpaYNaS9WqKCsnn73wmZb/EvCkj1OTUDQoKkROUWIX2guHpP50uWL6F6+6vCrAquequdew+2Nhhv0Ki+xQUIllQNqyUR+ZNynh1sXb7HkYueEJXFwFId0vT84acopA4DgoKkSyqmDGan112cto5F82t6ujuadfdX8WC5Vs67O9fXKCZ18dJQSHSAzqbc7G5fj9jb3tKVxdZlm4xR4CfXz8x4Gryh4JCpIesq7go7VpREFsvasHyLZx/x280MioLFq3aljYkFs4cpxFOJ0BBIdKDFs8ez1uLLk97Bz2A9w83MWvJejVHnYDr7q9KOQQWdBOibFBQiARgxfzJLJw5LuM5m+v3c44WGOy2q+5al/ZKYsqogeq8zgIFhUhA5pQP5bEbJzHmjAFpz3G0wGB3pJsnAbGQeOj68oAryk8KCpEAlQ0rZdXfTGHhzHFYhvO27z7EuNufUt9FGtW1DYz5/pMp50lArLlJIZE9CgqREMwpH8ofF13OqSelvxvxgY9amLVkPWNv1+ioZItWbWPWkvUcbmrtcKykuJDHbpyk5qYsU1CIhGjz7V9i3pQRFBWkv744+FFsdNSY7z/Z668wMnVaAzx4fblGN/UABYVIyCpmjGbHwhnMmzKCwgztUYebWpm1ZH2vbZL6/J3PpO20Bi3y15MUFCIRUTFjNG/+OPNQWjjWJNVb1o6qrm3gnIon0vZHlBQXsnDmODU39aD0DaQiEooV8yezrKqO76/YQounP2/t9j0Mr3iCUaefzDM3Tw2sviBNXvQc9fuOpD1+1Xlnsnj2+AAr6p0UFCIRNKd8KHPKh2ZckiJh++5DDK94gn5FBXxz0vC8+GSd6t7WHc5RSARGQSESYYkhnjdVvsKKzTsznnukuZWla2uo3FjH5tu/FER5WdeVgAAtFR40BYVIDlg8ezyLZ4/v0h/SfR82M7ziCQz43JBTcmLF1FgQHur0vJLiQl6947IAKpJkCgqRHJL4o9+VPgwntizI8Ion6FNoXP/Fc5jYP5g6u6orV0oQG3Vz58xxWrMpJAoKkRyU6MNYtGob962robnj3LM2mlqcpWtrWAqc/PxTfH3isNCabpZV1XHHr1/jSIoJc6loKY7wKShEcljFjNFUzBjNsqo6bnt8S6eBAXCosSUWGvGJa30KjcvHndGjHcOLVm3jFy/VcuBIS5f/n0KDR+ZN0tyICFBQiOSBxBXGsqo6/m319oxDSttranFWbN7Jis076VMITS1QYDB55PF9kj+eGpIVFxo/uGKsmpkiREEhkkcSgQGxmczpJqml0xT/wN/qx+ZpBGVgX9j0j5cH9nrSdQoKkTy18dZLgNjM5luXb+H1XQdozdD5HZbTS4rZeOslrFmzJuxSJA0FhUieKxtWypM3TQHgvuXP8fPtBdS9f5iwM0Od1LlDQSHSi4wsLeSFf5jaZt9Nla/wxO/fodWdVqfHAuSkogK+/+efUd9DDgolKMzsGuAHwGjgAnfflOa8t4ADQAvQ7O4TgqpRpLdITOZLWLRqG/f+V80JNVMVGFzxOS2xkS/CuqJ4FfgycE8Xzp3m7pkXuxGRrEkMuRVJCCUo3H0bgFmmm0GKiEgURL2PwoHfmJkD97j7velONLO5wNz45kEze73dKQOBqF6ZqLbui2pdoNqOV1Rri2pdkN3ahqU70GNBYWbPAoNTHLrF3R/v4tNMdve3zewTwDNm9gd3X5vqxHiIZAqSTVHt41Bt3RfVukC1Ha+o1hbVuiC42nosKNz94iw8x9vx/75nZsuBC4CUQSEiIj0jsrdCNbOTzWxA4jFwKbFOcBERCVAoQWFmM82sHvgC8ISZPR3ff6aZrYqfNghYZ2a/A14CnnD3p07gZdM2S0WAauu+qNYFqu14RbW2qNYFAdVm7mHPzxQRkSiLbNOTiIhEg4JCREQyyrugMLPLzOx1M9thZhUpjvc1s4fjx6vMbHiEavuumW01s9+b2XNmlnZcc9C1JZ03y8zczAIZLtiVuszsK/Hv22tmtiyIurpSm5kNNbPVZvZK/Gc6I6C6HjCz98ws5eAPi/nXeN2/N7Pzg6iri7X9RbymLWa23sw+F5Xaks77vJk1m9nVUanLzKaa2eb4e+CFrBfh7nnzBRQCbwIjgGLgd8CYduf8NbA0/ng28HCEapsG9I8/vjFKtcXPG0BsePIGYEIU6gJGAa8ApfHtT0Tle0aso/HG+OMxwFsB1TYFOB94Nc3xGcCTgAETgaog6upibZOSfpbTo1Rb0s/9eWAVcHUU6gJOBbYCQ+PbWX8P5NsVxQXADnevcfdGoBK4st05VwIPxh8/Clxkwawl0mlt7r7a3Q/HNzcAQwKoq0u1xf0Q+AlwfLcu65m6/gq4290bIDbnJkK1OfCx+ONTgJ1BFOaxSanvZzjlSuAhj9kAnGpmZ0ShNndfn/hZEux7oCvfN4D/ATwGBPV71pW65gC/cve6+PlZry3fguIs4E9J2/XxfSnPcfdmYD9wWkRqS3Y9sU99Qei0tnjzxNnuHtwtz7r2PTsXONfMfmtmG8zssgjV9gPga/Gh4KuI/ZGJgu7+LoYlyPdAp8zsLGAmsCTsWto5Fyg1szVmVm1m12X7BaK+1lOvZGZfAyYAF4ZdC4CZFQD/B/hmyKWkUkSs+WkqsU+fa81snLvvC7OouGuBn7n7v5jZF4Cfm9lYd28Nu7CoM7NpxIJicti1JFkM/C93b43YgqZFQBlwEXAS8KKZbXD3N7L5AvnkbeDspO0h8X2pzqk3syJiTQJ7I1IbZnYxcAtwobt/FEBdXaltADAWWBN/gwwGVprZFZ7mXiIB1QWxT8NV7t4E/NHM3iAWHBt7sK6u1nY9cBmAu79oZv2ILeIWWLNFGl36XQyLmX0WuA+Y7u5BvDe7agJQGX8PDARmmFmzu68ItarYe2Cvux8CDpnZWuBzQNaCIt+anjYCo8zsHDMrJtZZvbLdOSuBb8QfXw087/EeoLBrM7PxxO7RcUWAbe2d1ubu+919oLsPd/fhxNqOezokOq0rbgWxqwnMbCCxy/CaHq6rq7XVEfuUh5mNBvoBuwOorTMrgevio58mAvvd/Z2wi4LYSDHgV8DXs/mJOBvc/Zyk98CjwF9HICQAHgcmm1mRmfUHyoFt2XyBvLqicPdmM5sPPE1sdMID7v6amd0BbHL3lcD9xJoAdhDrIJododr+N1AC/Gf8U0udu18RkdoC18W6ngYuNbOtxO6E+PdBfArtYm03A/9uZn9LrGP7m0F8KDGzXxILz4Hx/pHbgT7xupcS6y+ZAewADgPf6umaulHbbcT6DP8t/h4I7M6WXagtFJ3V5e7bzOwp4PdAK3Cfu2d1XTwt4SEiIhnlW9OTiIhkmYJCREQyUlCIiEhGCgoREclIQSEiIhkpKEREJCMFhUgI4pPd9P6TnKBfVJGAmNnw+P0rHgJepe0yGiKRpQl3IgGx2E2yaoBJ8eW9RXKCrihEglWrkJBco6AQCdahsAsQ6S4FhYiIZKSgEBGRjNSZLSIiGemKQkREMlJQiIhIRgoKERHJSEEhIiIZKShERCQjBYWIiGSkoBARkYz+P+n4jrwk/eUYAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -124,12 +140,12 @@ "name": "stderr", "output_type": "stream", "text": [ - "100%|████████████████████████████████| 11/11 [00:00<00:00, 357.68it/s]\n" + "100%|█████████████████████████████████| 11/11 [00:00<00:00, 81.40it/s]\n" ] }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAEGCAYAAAAgxE+CAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA4yUlEQVR4nO3deXxU9d0v8M93JmQhbAlJBrJgEkjIgto2eQStQhB8Liq4AYppUQRFi3rLYpXepyp6by/Yi/ZRllqKCtqCtGqpFYxVaRAroGiFQCABAlkhYQmBBLLMzO/+MTMwmcxkmUxmzkw+b155McvJyTeTmfnM75zfOV9RSoGIiIh6ns7XBRAREfUWDF0iIiIvYegSERF5CUOXiIjISxi6REREXhLk6wJ6QlRUlEpMTGx1W0NDA8LDw31TUAdYW9dptS6AtblLq7VptS6ge7UVVNa1um6sq4HpYp14oi5yLSBDNzExEXv27Gl1W35+PnJycnxTUAdYW9dptS6AtblLq7VptS6ge7UlLt7S6vqJ9fO7XxB1iJuXiYiIvIShS0TUyziOcsl7GLpERERewtAlIiKAZwT2CoYuERGhufrIt76uoTdg6BIREXkJQ5eIqBfhJCrfYugSERF5CUOXiIjISxi6RES93PFlt/u6hF6DoUtE1Etwf67vMXSJiIi8hKFLRETkJQxdIqJejPtzvYuhS0TUC3B/rjYwdImIiLyEoUtEROQlDF0iol6K+3O9L8iXP1xE3gQwGUCNUmqUk/sFwKsAbgNwEcAspdR33q2SiMjPLRmIYyFXrppMwAjjBt/V04v5eqS7DsCkdu6/FUCK9WsugN95oSYiosCxZCCUtVeuiOV/vR4oCc71XU29mE9DVyn1BYCz7SxyJ4C3lcUuAINEZKh3qiMi8nNLBgKwhK0tcG2XRa7cT97j65FuR+IAlNtdr7DeRkRE5HdE2bY7+KoAkUQAH7nYp/sRgGVKqS+t1z8H8IxSao+TZefCsgkaBoMh69133211f319Pfr16+f5X8ADWFvXabUugLW5S6u1abUuoOPaxuXfCWnn+xWA7Tl/AwCMHz/+W6VUtmcrJEc+nUjVCZUAEuyux1tva0MptQbAGgDIzs5WOTk5re7Pz8+H421awdq6Tqt1AazNXVqtTat1AZ2oLR9Q6sqmZUcCtPv93377bUxQUNBaAKOg/S2jWmEGsN9oND6clZVV43in1kP3QwBPiMi7AEYDqFNKnfBxTUREvUJQUNDaIUOGpEdHR9fqdDrfbhb1E2azWU6dOpVx8uTJtQDucLzfp59cRGQjgJ0ARopIhYjMEZHHROQx6yJbAZQAOALgDwDm+ahUIiL/s6QOSuHyF9D6MpbUdbSGUdHR0ecZuJ2n0+lUdHR0HSxbB9rw6UhXKXV/B/crAI97qRwiooCT3LwBR4Jyodej9aFDHQcuAOgYuF1nfcycDmq1vnmZiIjcZGtyMMK4ATBeuZ1novId7hgnIiLPKNkejg33JWHV6HRsuC8JJdvDu7vKsrKyoMmTJycnJCSMyszMTB83btyIffv2hRQVFQWnpKRkeqJsR5cuXZLbb789ediwYaOuueaatKKiomBPrZsjXSIi6r68X8Ziz5sGGJt0gAJOF4WhJH8QsmdXY9LSKndWaTabcccdd4zIzc0989FHH5UAwM6dO8Oqqqr6JCUlNXv2F7ji1VdfjRo4cKCxrKxs/5o1ayIWLlwYv2XLlhJPrJsjXSKiXqRHNi2XbA+3BG6jJXABQCmBsVGHPW8Y3B3xfvTRR/2DgoLU008/fcp22/XXX39p0qRJ9fbLFRUVBWdlZY3MyMhIz8jISP/000/DAaC0tLRPdnb2yLS0tIyUlJTMvLy8fkajEVOnTk1MSUnJTE1NzXjhhRdinPzcQbNnzz4DAA899FDtV1991d9sNrvzK7TBkS4RUQDyatP6XatjLCNcJ0zNgl2rY5A87lhXV7tv376wa6+99mJHy8XGxhp37NhR3LdvX1VQUBBy//33J+/fv//gm2++GTlhwoS6l1566aTRaMSFCxd0O3fu7HvixIk+hw8fPgAAp0+f1juur7q6Otg2ku7Tpw/69etnqq6uDho6dKjRcdmuYugSEVH31B4PvTzCdaSUWO7vOc3NzTJnzpyrCgsLw3Q6HUpLS0MAYMyYMQ2PPvpoYktLi27atGm1N9xww6W0tLSm8vLykAcffDBhypQpdXfffff5nqzNETcvExEFGK+OcgEgIrERIs5TV0QhIrHRndVeffXVl/bu3du3o+V+/etfG2JiYloOHjxYWFBQUNjS0qIDgFtvvbX+iy++KIqLi2uePXt20sqVKwdHR0eb9u/fXzh+/PgLr7/+evSMGTMSHddnMBiajx07FgwALS0tqK+v1xsMhm6PcgGGLhFRQGkvcHvsUKEx82qgD3EeuvpghTHz2pwOsTOmTJlyobm5WZYvXx5lu2337t1heXl5rU44XVdXpx86dGiLXq/H6tWrB5tMJgBAcXFxcHx8fMuiRYtOP/DAA6e+++67vidOnAgymUyYNWvWuaVLl1YWFBS0CfXbb7/93JtvvjkYAN56662I66+//oJO55m4ZOgSEQWIjka4PTYCTh7XgOzZ1QgKMV8e8YooBIWYkT2nGsnjGtxZrU6nw4cffnh027ZtAxISEkaNGDEi85lnnomLi4trsV9u/vz5NRs3bhw8cuTIjEOHDoWGhYWZAeCTTz7pn56enpmenp7x/vvvRz799NPVx48f73PjjTeOTEtLy5g5c2byiy++WOH4c3/+85+frq2tDRo2bNioFStWDFm+fHmbZdzFfbpERNR9k5ZWIXVSHXatjkHt8VBEJDZizLwadwPXJjExsWXr1q1OD9exTYa6+uqrm4qLiwttt//ud7+rBIAnn3zyzJNPPnnG8fsKCwsPtvcz+/btqz7++GOPHCLkiKFLRESekTyuwZ1Zyr0JNy8TERF5CUOXiIjISxi6REQBoqPZyWx04HsMXSKiXoCBqw0MXSKiAMfA1Q6GLhFRgPD6magc7D6xO/zxzx9PunPznemPf/540u4Tu/2ytd/HH3/cLyMjIz0oKCjrrbfeivDkunnIEBERddtLX78U+5fivxiaTc06BYVjdcfCdp/YPWh66vTqZ657xq9a+yUnJze/9dZbx5ctW2bw9Lo50iUiom7ZfWJ3+F+K/2JoMjXplLXxgYKSJlOT7s/Ffza4O+L1VWu/kSNHNo8ePfqSp079aI8jXSKiAOaN/blvF74d02xqdppQLaYWebvw7ZjRQ0f7TWu/nsTQJSIKAL7cn1txoSJUuWjtp6Cksr6Srf2suHmZiIi6Jb5/fKPAeWs/gaj4fvF+1dqvJzF0iYj83JB/fo/G/4y98jXB4/N/2vVAxgM1wfpgp6HbR99HzcyY6Vet/XoSQ5eIyI8N+ef3gLLmnYjlf70ejbcM9drxuaOHjm6Ynjq9OlgfbLaNeAWigvXB5ntT760ePXS0X7X22759e1+DwXDN1q1bIxYsWHDViBEjPHZoEvfpEhH5qRnKOuCzha3D5SH//B4nx//AK7U8c90zVTkJOXVvF74dU1lfGRrXL67xgYwHatwNXBtftPYbN27cxerq6n3dqdsVhi4Rkd/y6sTbDo0eOrrBnVnKvQk3LxMRBSL70S9pBkOXiMifKeeH6pA2MXSJiPyWydcFUBcxdImI/NS7Um8Z6dq+gFaXvTWJijrPp6ErIpNEpEhEjojIYif3zxKRUyLyvfXrYV/USUSkVaGfngCsx6XaHzrEwNUmn81eFhE9gFUAbgFQAeAbEflQKVXosOgmpdQTXi+QiEjjZuVZjsYJ/by61e2+6p/bsHNX+Nn162Oay8tDgxMSGiMffLAm/Pox3TpkqKysLGjevHnD9u7d23fAgAGmqKiolhUrVpSHhISoyZMnp9gOG/KkJUuWGN55550ovV6vBg8ebFy/fv3x1NRUj3Q18uVI9zoAR5RSJUqpZgDvArjTh/UQEZGbTv7fpbHljz2WWr99e2Tz0aN967dvjyh/7LHUk/93aay767S19hs7duyF8vLy/QcOHDi4bNmyyqqqqj6erN1RVlbWxe+///5gcXFx4V133VW7YMGCeE+t25fH6cYBKLe7XgFgtJPlporIWADFABYopcqdLAMRmQtgLgAYDAbk5+e3ur++vr7NbVrB2rpOq3UBrM1dWq1Nq3VZKABtDw3ydr0NO3eFn9u0yaCamq4M5JQS1dQk5zZtMvQfP77OnRGvq9Z+gKWdn+22oqKi4Nzc3KRLly7pAODVV18tu+WWWxpKS0v7TJ06Nbm+vl5vMplkxYoVpRMnTqy/7777Evft2xcuIuonP/nJ6eeff77VaSqnTJlywXb5xhtvrN+0adPgrtbuitZPjvF3ABuVUk0i8iiA9QBudragUmoNgDUAkJ2drXJyclrdn5+fD8fbtIK1dZ1W6wJYm7u0WptW67J0FWobuL7YtHx2/foY1ey8tZ9qbpaz69fHhF8/xi9b+/3+97+PnjhxYl1Xa3fFl6FbCSDB7nq89bbLlFL2p+9aC+A3XqiLiIi6oLm8PNTl8cJKSXNFhV+29lu9enXk3r17+/7+978v8lStvtyn+w2AFBFJEpFgADMAfGi/gIgMtbt6B4B2z5dJRETeF5yQ0Ahx3toPIio43v9a+23evLn/8uXLh27duvVIWFiYx85A4rORrlLKKCJPAPgElhOIvqmUOiAiLwLYo5T6EMD/FJE7ABgBnAUwy1f1EhFpxefbhmPtLVeum0zAo9te89ms5cgHH6xp2LlzkGpqarO9W4KDVeSDD7rd2u/ZZ5+V5cuXRz311FOnAUtrv9raWn1SUtLl2cR1dXX6+Pj4Zr1ej5UrV7Zq7ZecnNy8aNGi001NTWJt7VcXEhJinjVr1rnMzMzGmTNnJjv+3H/9619hTz755FVbt249HBcXZ3Sndld8uk9XKbUVwFaH256zu/xLAL/0dl1ERFr1+bbhrTr5KQXo9cAfJv5PAL4J3fDrxzQMuu++6nObNhlUc7NAKYGIkuBgNei++6rdPWzI1tpv3rx5Ca+++uqQkJAQFR8f37RixYpWE2rnz59fM3Xq1OHvvvvu4JtvvrnOvrXfa6+9NiQoKEj17dvX9Kc//enY8ePH+8yZMyfRbDYLADhr7feLX/wi4eLFi/rp06cPB4DY2Njmbdu2HXHnd3Ck9YlURERk9fm24QBcdvLD59uGY8LNR71clcWQ//XLqv7jx9edXb8+prmiIjQ4Pt4jx+n6orXfV199VdydmtvD0CUiIo8Iv35MgzuzlHsTnnuZiCgAsJOff2DoEhH5EXby828MXSIiIi9h6BIR+YkJNx912snP/n7SNoYuEZGfSFy8BY989lqbTn4AA9dfcPYyEZGfeXTba62u++qkGI4qDp0N//7z8pjzpy+FDogKa/zBhISa+LRIv2vt95vf/CZ67dq10TqdDuHh4aY1a9aUZmVluXVWLUcMXSIi6rYdfy6OPbCjymBqMesAoPbkxbCKQ7WDMm+Krb7p3tQqd9Zpa+2Xm5t75qOPPioBgJ07d4ZVVVX1sT8jlac9/PDDZ2ydjf70pz8NnD9/fsKOHTsOe2Ld3LxMROQHLF2F2tLCKLfi0Nlw+8AFACiIqcWsO/BFlaHi0Nlwd9brqrXfpEmT6u2XKyoqCs7KyhqZkZGRnpGRkf7pp5+GA0BpaWmf7OzskWlpaRkpKSmZeXl5/YxGI6ZOnZqYkpKSmZqamvHCCy/EOP7cyMhIs+1yfX29Xjx4PBZHukRE1C3ff14e0ypw7ZhMZvn+8/KY+LRIv2rtt3Tp0ujVq1cbWlpadJ9++mlAdBkiIqIAcP70Jdet+xSk3fs9oLm5WXJzcxNTU1Mzpk+fPvzo0aOhgKW138aNG6MWLlwY+/XXX4dFRESY7Vv7vffeewMiIiJMztb5y1/+8lR5efn+JUuWVDz//PNDnS3jDoYuEZHGudq0vG6SW1ttPW5AVFgjBC5a+0ENiArzu9Z+No888sjZTz/9dJA79TvD0CUiom75wYSEGn2Qzmno6vU69YMJCW639mtubpbly5dH2W7bvXt3WF5eXj/75erq6vRDhw5t0ev1WL16davWfvHx8S2LFi06/cADD5yytvYLMplMmDVr1rmlS5dWFhQUtAn1goKCENvlTZs2Dbzqqqua3KnfGe7TJSLSsCVLluDBkCvXTSbgj8b/8F1BTsSnRTZk3hRbfeCLKoPJZBYoCARKr9epzLGx1e4eNuSr1n6vvPJKzI4dOwYEBQWpgQMHGtetW+exJg4MXSIijVqyZMnly/a9cx/QfYMXX1yC/Px8H1XW1k33plYlXRNV5+njdH3R2u+tt94qb+/+7mDoEhFpkGPg2v9vuz8nJ8erNXUkPi2ywZ1Zyr0J9+kSERF5CUOXiMjPsHeu/2LoEhFpFHvnBh6GLhERkZcwdImINGjJkiVOe+faLttPtCL/wdAlItKgxMVb8Hbzf7TpnSui3cAt2783/K8vvZC0btHP0v/60gtJZfv3dvuUWWVlZUGTJ09OTkhIGJWZmZk+bty4Efv27QspKioKTklJyfRE3a6sW7dukIhkffHFFx2eFauzeMgQEZGG/dH4H4DxynUtdBVy5p/r/hC797OPDSZjiw5K4UxlRVhpwd5B1068tXr8rEf8qrUfANTW1upWrlxpuOaaa7p1nLEjjnSJiKhbyvbvDd/72ccGU0uzzm5buJhamnV7P9tqcHfE66vWfgCwaNGiuKeeeupkSEiIR6ezcaRLRKQxWu6d68y3WzbHmIwtzlv7GY3y7ZbNMcNGXes3rf2+/PLLvpWVlcEzZsyoe+WVV4Z0te72MHSJiKhb6mpOhro8vkkpqaup7vHWfnPmzLmqsLAwTKfTobS0NASwtPZ79NFHE1taWnTTpk2rveGGGy7Zt/abMmVK3d13333efl0mkwkLFy5MeOedd3rkzFrcvExERN0yMGZII0RctPYTNTDG4Det/c6dO6c/fPhw6M033zwyLi7u6r1794ZPmzZthKcmUzF0iYg0xN82LQNA1u131eiD+jhv7RcUpLJuv8tvWvsNHjzYVFtbu7eysrKgsrKy4Nprr2147733jowdO7bDzdydEfCbl5vKzuPU6r0YDh0q8nY4XSZ+2U1eroqIqK2KxTuwA/0vXzfBhBx45L2+Rw0bdW3DtRNvrd772VaDyWgUKCUQUfqgIHXtxNuqh4261q9a+/Ukn4auiEwC8CoAPYC1SqllDveHAHgbQBaAMwDuU0od7+z66z4txYXPyyzrguuTlVYs3sHgJSKfqlh8ZVAgECgo6KHHF+iHsahv5zu1YfysR6qGZ19X9+2WzTF1NdWhA2MMjVm331XjbuDa+KK1n72vv/66qOtVu+az0BURPYBVAG4BUAHgGxH5UClVaLfYHAC1SqkRIjIDwEsA7uvM+pvKzl8O3I4oKAYvEfmMY+Da/w8AX2KA12tyx7BR1za4M0u5N/HlSPc6AEeUUiUAICLvArgTgH3o3glgifXyewBWiogo1fFpwOv/1bVjsRWUy30p9rS8X4WIiLTNl6EbB8B+u3wFgNGullFKGUWkDsBgAKcdVyYicwHMBQCDwYDaYzUIbWeTcqvvtW7K6QxnwbxuUvfOdFZfX4/8/PxuraOnaLU2rdYFsDZ3abU2b9Q1HDqXu8Bs70/OatDqY0auBcxEKqXUGgBrACA7O1tFJMXg0t5THXyX9Xs7GbiuzMpru8uiKyPi/Px85OTkdKuGnqLV2rRaF8Da3KXV2rxRV0XeDiiodoPXWQ1afczINV+GbiWABLvr8dbbnC1TISJBAAbCMqGqQ/1+HNvp0AUsswQ9yXFEzM3SRETky+N0vwGQIiJJIhIMYAaADx2W+RDAg9bL0wBs68z+XAAIGTYA/ScMa3cZZfevp6flJy7ecvmLiMhe/LKbWr0fAWh1mZM8A4fPRrrWfbRPAPgElkOG3lRKHRCRFwHsUUp9COANAO+IyBEAZ2EJ5k4beMtVCB0ZgVOr9zrddGO7nrBsLI63sx5PB6X9+jgCJiLbe0I++kIP/eWwFYhfBW7jkXPh9V9WxhjPNoYGRYY29rsxriZ0xKBuHTJUVlYWNG/evGF79+7tO2DAAFNUVFTLihUrykNCQtTkyZNTbIcNedJrr702+Pnnn483GAwtADB37tyahQsXtplL5A6f7tNVSm0FsNXhtufsLjcCmN6dnxEybADil93UrX0fzoLRU0F8eT15WxjARL2c4xY3f3pPOPf3o7H1u08YYFQ6ADCeuhjWeKR2UL/RQ6sHTRnud639pkyZUvv222937rjTLnAZuiKyCsAGpdS/PP1DA0FPBLHt+/3phUZE1HjkXLh94AIAFARGJfW7TxpC0wfXuTPiddXaD7C087PdVlRUFJybm5t06dIlHQC8+uqrZbfccktDaWlpn6lTpybX19frTSaTrFixonTixIn19913X+K+ffvCRUT95Cc/Of3888+7dZpKd7Q30i0GsFxEhgL4M4CNSql/e6cs/+QYlu6GMDc/E/Ue/niuZUf1X1bGtApceyaz1H9ZGRM6YpDftPYDgI8//nhQampqv+Tk5MaVK1eWjxgxoqWr9TvjciKVUupVpdT1AMbBMmP4TRE5JCLPi0iqJ354oDu+7PbLX+7i5Csi0jrj2UbXrfsUpN37PaC5uVlyc3MTU1NTM6ZPnz786NGjoYCltd/GjRujFi5cGPv111+HRUREmO1b+7333nsDIiIi2hy6cu+9954rKysrKC4uLpwwYcL5n/70p0meqrXD2ctKqVKl1EtKqR8CuB/AXQA6fd5KsuhuADN8iUirgiJDGyEuTnggUEGRIX7T2g8AhgwZYgoLC1MAsGDBgtMHDhzwSFs/oBOhKyJBIjJFRP4E4GMARQDu8VQBvVF3ApiHHhEFjkDYtAwA/W6Mq4HeRT9dvU71uzHeb1r7AUBpaWkf2+UNGzYMSk5OdutDgzPtTaS6BZaR7W0AvgbwLoC5SqluTf+m1o4vux35+flOz2rVEU68IiItCB0xqKHf6KHV9btPGmAyi/UITQW9TvUbPaTa3cOGfNXa7ze/+U3MJ598Mkiv16tBgwYZ161bd9yd+p1pbyLVLwFsALBIKVXrqR9IztkHZ1dHsS/dNxl6oNVRyIs2feSZwoioR7x832Q8YXfdBOB3ST/zVTndNmjK8KrQ9MF1nj5O1xet/VatWlWJtmdI9AiXoauUurknfiB1zBbAnQnfx4/9rlXYKljC9+X7JjN4iTTq5fsmX94BKrC8bvWwvJ5/4cev29ARgxrcmaXcm/jyNJDUgY72+/7MGri2L9j9r2B5YRORtthel46vW9sXX7eBLWC6DAUyV5uenR5cZqd7vZOIqKfYtkhR78ORrp9xHP26euHabudMZyLt6eh1S4GLoeunji+7/fK+IGccb2f4EmkHt0L1XgxdP9feJ2NnHYIZvkS+ZfR1AeRTDF0/5mx2srL7au/wAwYvkW/8LulnrV6nQOuRrz8fdVBSUhK+YcOGpFWrVqVv2LAhqaSkJLy76ywrKwuaPHlyckJCwqjMzMz0cePGjdi3b19IUVFRcEpKSqYn6nZm7dq1EcOHD88cMWJE5pQpUzx2GkhOpPJzizZ91Gq2o8DySbozx/vx5BpEvrEq6Wf42bHfQY/ACdy8vLzYPXv2GIxGow4ATp8+HVZSUjIoOzu7etKkSX7V2q+goCDk5ZdfHrpr165D0dHRpsrKSo9lJUM3ADh7oT6Dzo9mGb5E3mH/mrT/YOzvr72SkpJw+8AFAKWUGI1G2bNnjyE1NbUuOTnZb1r7rVq1KvqRRx6piY6ONgFAXFycx/YKMHQDWFdOsmFbzt9f/ETkfbt27YqxD1x7JpNJdu3aFZOcnOw3rf2OHDkSAgA/+tGP0kwmE5599tmqadOmne9q/c4wdHuBroQvR71EPSOQ51HU1ta6bN2nlJL27veE5uZmmTNnzlWFhYVhOp0OpaWlIYCltd+jjz6a2NLSops2bVrtDTfccMm+td+UKVPq7r777jZhajKZ5OjRoyE7d+4sOnbsWJ+cnJy0nJycA1FRUc7mp3YJJ1L1Il0J0kB+gyDSkkD4gBsREdEo4rzLkIioiIgIv2rtN3To0ObJkyefCwkJUWlpac1JSUmNBw4cCHHnd3DE0O1lutJSMHHxFre6HxFR7zJmzJgavV7vNHT1er0aM2aMX7X2u+eee85t3769PwCcOHEi6NixY6EjR45scud3cMTQ7aW6Gr5E5L5A6ZvrSnJyckN2dnZ1UFCQ2TbiFREVFBRkzs7OrnZnEhVwpbXftm3bBiQkJIwaMWJE5jPPPBMXFxfXYr/c/PnzazZu3Dh45MiRGYcOHQq1b+2Xnp6emZ6envH+++9HPv3009XHjx/vc+ONN45MS0vLmDlzZrKz1n733HPP+cjISOPw4cMzx40bl/riiy+WDxkypNublgHu0+31ji+7vVOh+upj/4AeeojD6Tgef53NqIjas+qxbXgKV3ZpmmDCbwe1tPMd/mnSpElVqampdbt27Yqpra0NjYiIaBwzZkyNu4Fr44vWfjqdDmvXrq0A0CaQu4uhSx1OtFp0LqRN2Nqsemwbg5fIhVWPbYOyHolr6equoIcei87p8PIgj2yt1JTk5OQGd2Yp9ybcvEyXOdvUteBcH4jdP2dWPbatp0sj8ju214X9a8f+tfSLc2G+LI98hKFLrTju69V32ECQiIg6i6FLTrVuH9h+wzHFnilEXdLRa4oCF0OXXLK0D5R2Q9V2H2c4E7XFD6TkiKFLHeroU7nJ2kSQwUt0hZFN/MgJhi61K3OG86eIsvtnf/gD+/USWfx2UEur1wnQeuQbiLP+z579Kvz7vQ8n7dr1P9K/3/tw0tmzX/lla785c+YkpKWlZaSlpWUkJiaO6t+//w88tW4eMkQdevz1m1vNULa9cbR3vCGbJxABLw9qwoJzfaCHPuADt7j4/8RWVm0wmM3NOkCh4eLRsNrarwbFxeZWp6b+yq9a+73xxhvltsu//vWvY77//vsOT0XZWT4JXRGJBLAJQCKA4wDuVUrVOlnOBKDAerVMKXWHt2qk1py9SXQ0omXwUm9l/9qwfDC1fDgN1NfD2bNfhVsCt8lu05gSs7lJKqs2GKKibq6LjLzBb1r72Xvvvfcin3vuObc+NDjjq5HuYgCfK6WWichi6/VnnCx3SSn1A69WRp3WmbNZsWsRUeArK38zxjLCbctsbpay8jdjIiNv8JvWfjbFxcXBFRUVwVOmTPFIWz/Ad/t07wSw3np5PYC7fFQHdRPP30zUWm98rjdeKg+Fy5naShovVfR4a7/c3NzE1NTUjOnTpw8/evRoKGBp7bdx48aohQsXxn799ddhERERZvvWfu+9996AiIgIl+dUXr9+feRtt91WGxTkufGpr0a6BqXUCevlkwAMLpYLFZE9AIwAlimlNrtaoYjMBTAXAAwGA/Lz81vdX19f3+Y2rfD32tZNssyV6KgjUeLiLZeX9UZdvsLa3KPV2jxV17pJ4R7//bTymIWGJTQ2XDwaBignhzqICg1LcLu13+bNmyM6Ws7W2u/9998/ZjabERYWlgVcae33/vvvD5w9e3bSE088Uf3EE0+c2b9/f+Ff//rXAa+//nr0pk2bIv/yl78cd7beDz74IPK1114rdad2V3osdEXkMwBDnNz1X/ZXlFLKVR9GAFcppSpFJBnANhEpUEoddbagUmoNgDUAkJ2drXJyclrdn5+fD8fbtCJQajue0/GnfFswd3dzc6A8Zt7G2rquK3W19/zvid9NK4/ZsITZNbW1Xw0ym5vahK5OF6yGJTzkdmu/Z599VpYvXx711FNPnQYsrf1qa2v19hOp6urq9PHx8c16vR4rV65s1dovOTm5edGiRaebmprE2tqvLiQkxDxr1qxzmZmZjTNnzkx29rP//e9/h54/f14/YcIEj/Y37bHNy0qpiUqpUU6+/gagWkSGAoD1f6d/EKVUpfX/EgD5AH7YU/WSZ3BzM1FbgT6nITLyhoa42NxqnS7EDNgGUaJ0uhBzXGxutTuTqADftfYDgHfeeSfyzjvvPKvTeTYmfbV5+UMADwJYZv3/b44LiEgEgItKqSYRiQLwYwC/8WqV5JaOuhbZcHYzBZLe/kEyNfVXVVFRN9eVlb8Z03ipIjQ0LL5xWMLsGncD18YXrf0A4JVXXvHYjGV7vgrdZQD+LCJzAJQCuBcARCQbwGNKqYcBpAP4vYiYYRmRL1NKFbpaIWlPR7Ob/775KRRufgoAWp3zKv1Qh68HIk05mJaOrXbXTQCm3LUcQOCPcu1FRt7Q4M4s5d7EJ7OXlVJnlFITlFIp1s3QZ62377EGLpRSXymlrlZKXWv9/w1f1Erd4+oNZ8vmpy73L3LcCXQwLb1HayLyJPvnq+25rIflOU7kiKeBpB7n2C7w75ufggCXv5xh8JI/cBa49s/tjxm85IChS15jC1526CWi3oqhS15lC96OuomyIRr5O3bMJWcYuuR1nQ3c3j4blPwDPyBSVzB0SZNs52Vj8JKWsWNuaztqL4TP3FeSNHb3wfSZ+0qSdtRe8MvWfocPHw4ePXp0anp6ekZqamrGpk2bBnpq3Qxd8jpXhwQpuy/b4RYAg5e0a8pdy1s9b4HWI9/edPjbs4crYn+6ryT1szPnI4svNvX97Mz5iJ/uK0l99nBFrLvrtLX2Gzt27IXy8vL9Bw4cOLhs2bLKqqqqPp6s3dFzzz039J577qk9ePBg4caNG0sWLlw4zFPrZuiSTzi+GdneqEwAbrcLXJvExVsYvqQptufj7Xctv7xlprcG7o7aC+FvV50xNJmVzu7DhzSZle7tqjMGd0e8rlr7TZo0qd5+uaKiouCsrKyRGRkZ6RkZGemffvppOACUlpb2yc7OHpmWlpaRkpKSmZeX189oNGLq1KmJKSkpmampqRkvvPBCjOPPFRGcP39eDwC1tbX6mJgY543D3cAm9uQzzt6UeBYr8kf2W2Z64/NzTfmpmGazcjqIazYrWVN+KuamiP5+09pv6dKlVbfcckvK2rVrYy5duqTbsmVLcVdrd4UjXdKUzrxhccRLvsbnYGull5pCXTb2A6TsUpNftfZ76623Iu+///4z1dXV+z744IPDs2bNSrI1Ueguhi5pTmeCt6M2gkS+0BtHuQBwVVhIo7iYyC2AGhYW4nZrv7179/btaDlba7+DBw8WFhQUFLa0tOiAK6394uLimmfPnp20cuXKwdHR0ab9+/cXjh8//sLrr78ePWPGjETH9f3xj3+Mmjlz5lkAmDhxYkNTU5Pu5MmTHtkyzNAlTersiJcjDvI2PufampsQXROsc96iNVgnam5CtNut/Zqbm2X58uVRttt2794dlpeX189+ubq6Ov3QoUNb9Ho9Vq9e3aq1X3x8fMuiRYtOP/DAA6esrf2CTCYTZs2adW7p0qWVBQUFbUI9Nja2eevWrQMA4Lvvvgttbm6WoUOHemSyOkOXNMvx9JGu8E2QtKC3jnIB4KaI/g0PxA6uDtGJ2TbiFUCF6MT8QOzg6psi+vtVa7/f/va35evWrYseOXJkRm5ubvLrr79+3FMt/jiRijSvo25FACdYkXfwA55r/zslvuo/owbWrSk/FVN2qSl0WFhI49yE6Bp3A9fGF639srKyGr/77rtD3anbFYYu+YXOBG/6G1cjyOEZXfBgQQ9WRb3F1euvBgD0S7NcN5mAS4eXXb6fH/gsboro3+DOLOXehJuXyW+098YWPnIx9HpAOexVsr1ZErnrydInAVieW2I9h6leb3nOEXUVQ5f8irP9vGEpiyGCy1+OGLzkLvvnju25Zf9cC0tZzFEudQlDl/zSuklXTnCjZ69A8hE+96irGLrkt+xHGM5GuEQ9ydWWFaL2MHTJr3Vm055SnHVKRNrA0CW/195ow35iFYOXukqptpPzyLV/HTkdPnvdN0m3vLI9ffa6b5L+deS0X7b2Ky4uDr7++utTU1NTM6677rqRR48e9VhXI4YuBTz7U6YyeKkrPHS63V7hxb8fiJ297pvUfx6qiTxcU9/3n0U1EbPXfZP64t8P+F1rv5///Ofxubm5Z4qLiwt/9atfVS1atCjeU+tm6JLfc3Ysrm2EolTr4ykBBi91TuLiLbh0eFmr5xLQeuTL48At/nXkdPifdpcZmozmK639FKTJaNb9aXeZwd0Rr69a+x0+fDjs1ltvPQ8AkydPvvDZZ58Ncqd+Zxi6FBCcvfmZTEBD0TInSzN4qfMaipZdHvHaH6vLwL3ijS+PxTQbzc5b+5nM8saXx9oEW2d0tbVfYWHhwU2bNpUsWLBgGADYWvsdOnSo8ODBgwdGjx590b61X3FxceHjjz/e5oxV6enpFzdu3BgBAO+8886ghoYG3cmTJz0yV51npKKA4exNsL1wtd3H4yzJkePzhmefal/52YuuW/spSPnZiz3e2m/OnDlXFRYWhul0OpSWloYAltZ+jz76aGJLS4tu2rRptTfccMMl+9Z+U6ZMqbv77rvPO65vxYoVFXPnzh2Wnp4eNWbMmAsxMTEtQY6nu3MTR7oU0NgwgbqqvecDA9e5hMi+jSIuWvsJ1LDIvn7V2i8xMbHlH//4x9GDBw8W/vd//3clAERFRXlkDz9DlwIeg5eoZ825MakmWK9z3tpPr1Ozb0zyq9Z+tmUA4Fe/+tXQ+++//7Q79TvD0KVegcFLncFRrnt+PCKq4Sejh1WHBOnMthGvCFRIkM78k9HDqn88IsqvWvvl5eX1T05OHpWYmDiqpqYmaOnSpSfcqd8Z7tOlXqOzLQJtyxLZ8PnQseemZFZNSDfUvfHlsZjysxdDEyL7Ns65ManG3cC18UVrv4ceeqj2oYcequ1O3a4wdKlXsb15sj8vOeKWju778Yiohh+PiGJrv3YwdKlX6mjUeyQoF+p5h7NdLanr+cLIu16MAswtUAo4FmK5yWQCRhg3XF7EvrkGUXf5ZJ+uiEwXkQMiYhaR7HaWmyQiRSJyRETYvJI8ytVItiQ4t1X3mMsnQ1gysOeLIu9ZMhAwW3YN2jcv0OstzwGinuCriVT7AdwD4AtXC4iIHsAqALcCyABwv4hkeKc86i0cg/dIUG6b3rytR7sM3oDwYpTTm+3/9keCcrmLgTzOJ5uXlVIHAUDa74t1HYAjSqkS67LvArgTQGF730TUVfb7edvrj2p/NiLyc+aWDhfx0LkQiFrR8tMqDkC53fUKAKNdLSwicwHMBQCDwYD8/PxW99fX17e5TStYW9f1RF3rJoUD/3QdrCKW4N3ewc/V6mMGsDabcQDa+/wkAihY/tZ8zMiTeix0ReQzAEOc3PVfSqm/efrnKaXWAFgDANnZ2SonJ6fV/fn5+XC8TStYW9f1WF35rke0tn27s/IsR0C42vSo1ccMYG1XfljHWy4EQE5ODh+zrijZHo5dq2NQezwUEYmNGDOvBsnjunXIUFlZWdC8efOG7d27t++AAQNMUVFRLStWrCgPCQlRkydPTrEdNuRJH3/8cb9FixYlFBcX9/3DH/5QYn/40IoVKwYvX758KAA89dRTJ5wdktSeHgtdpdTEbq6iEkCC3fV4621EPaq9N2LHNoHc5+efLhn1CNWzb59H5f0yFnveNMDYpAMUcLooDCX5g5A9uxqTlla5s0pba7/c3NwzH330UQkA7Ny5M6yqqqpPUlJSs2d/gSuSk5Ob33rrrePLli0z2N9eXV2tf+mll2K//fbbQp1Ohx/+8IcZM2bMOBcdHd3pJ5OWz0j1DYAUEUkSkWAAMwB86OOaKNA5OSzIvrWb/aEkAI/t9EeJi7cg3fhOm5Z9bfAQsc4r2R5uCdxGS+ACgFICY6MOe94woGS7X7X2GzlyZPPo0aMv6XStI3Lz5s0Dx44de95gMJiio6NNY8eOPf/BBx90aXalT/bpisjdAFYAiAawRUS+V0r9DxGJBbBWKXWbUsooIk8A+ASAHsCbSimPb0YgamNJXZtZyo7HbtrjWaz8U3LzBhwMmolQvely8F7eysHA7Zpdq2MsI1wnTM2CXatjkDyuyyfN6Gprv759+6qCgoKQ+++/P3n//v0Hba39XnrppZNGoxEXLlzQ2bf2A4DTp093umVfZWVln/j4+Msj7Li4uObKyso+XfmdfDV7+a8A/urk9ioAt9ld3wpgqxdLI7Kwe9MVACM6MaLl5mbtc9wykW58BzBaLvNv1w21x0PhvMmQZcRbe9yvWvv1JC1vXibSjOPLbmfTBD/HZgY9KCKxESLOU1dEISLRr1r7uRIXF9dSUVERbLteWVkZ7Nh8oSMMXaIu6Myb86y8BoavxvDv0cPGzKuBPsR56OqDFcbM86vWfq7cddddddu3bx9w6tQp/alTp/Tbt28fcNddd3VpXwRDl6iLOjsq4hu9NnT0d+Ao1wOSxzUge3Y1gkLMl0e8IgpBIWZkz6l297AhX7X22759e1+DwXDN1q1bIxYsWHDViBEjMgHAYDCYfvGLX1RlZWWlZ2VlpT/99NNVBoOhS9PgtXxyDCLN6kq3IvvlSVv4d/GgSUurkDqpztPH6fqitd+4ceMuVldX73N23/z588/Mnz+/S8fm2mPoEnVDZ3r0Apxk5Svcj+tlyeMa3Jml3Jtw8zJRN3FzszYxcEmLGLpEHtCV2c0M357Hx9hjzGazmW0+usj6mJmd3cfQJfIgjnp9jxOnPGr/qVOnBjJ4O89sNsupU6cGwtLCtg3u0yXysHWTwpGTk8NJVj7AwPUso9H48MmTJ9eePHlyFDhI6ywzgP1Go/FhZ3cydIl6SFcmWdmWJ/cxcD0vKyurBsAdvq4jkPCTC1EP6sobPTc5u4+BS/6CoUvUwzo7yQrgRCt3MHDJnzB0ibyEo17PY+CSv+E+XSIv6uyZrOyXYXC01ZnHj48baRFHukQ+wE3O7mPgkj9j6BL5UFc3Off28GXgkr9j6BL5WFdGvUDvDV8GLgUChi6RRnQ1MHpL+Hb292Tgkj/gRCoiDenKRCubQJ1w1dnHINB+bwpsHOkSaVBXNzkDgTXyZeBSoOJIl0jDetvItyu/pz/+fkQMXSI/0J3wtf9+rerqCF3rvw+RKwxdIj/iTvjaL79uUrjHa+oOhi31NgxdIj/kbvjOymsA8ny7+dmd/c4MWwoUDF0iP+Zu+Dr7np4Mtu5M8GLgUiBh6BIFgO6Er42z73Un8Dw1g5phS4GIoUsUQOyDyhPh54tDkBi2FMgYukQByhOjX29aNykcOTk5vi6DqEcxdIkCnKdHv55kX1t+fr7vCiHyEoYuUS/iOJrk5mMi7/JJ6IrIdABLAKQDuE4ptcfFcscBXABgAmBUSmV7q0ai3sBZAHoyiBmwRK35aqS7H8A9AH7fiWXHK6VO93A9RGTFoCTqOT4JXaXUQQAQEV/8eCIiIp/Q+j5dBeAfIqIA/F4ptcbVgiIyF8Bc69V6ESlyWCQKgFZHzKyt67RaF8Da3KXV2rRaF+DZ2q7y0HqoHT0WuiLyGYAhTu76L6XU3zq5mhuVUpUiEgPgUxE5pJT6wtmC1kBuL5T3aHWfMGvrOq3WBbA2d2m1Nq3WBWi7NnKux0JXKTXRA+uotP5fIyJ/BXAdAKehS0REpHWabWIvIuEi0t92GcB/wjIBi4iIyC/5JHRF5G4RqQBwPYAtIvKJ9fZYEdlqXcwA4EsR2QvgawBblFJ53fixLjc9awBr6zqt1gWwNndptTat1gVouzZyQpRSvq6BiIioV9Ds5mUiIqJAw9AlIiLykoALXRGZJCJFInJERBY7uT9ERDZZ798tIokaqm2hiBSKyD4R+VxEvHbcXEe12S03VUSUiHjlMIXO1CUi91oftwMissEbdXWmNhEZJiL/FJF/W/+mt3mprjdFpEZEnE48FIvXrHXvE5EfeaOuTtb2E2tNBSLylYhcq5Xa7Jb7DxExisg0rdQlIjki8r31NbDdG3WRm5RSAfMFQA/gKIBkAMEA9gLIcFhmHoDXrZdnANikodrGA+hrvfwzLdVmXa4/LIds7QKQrYW6AKQA+DeACOv1GK08ZrBMcvmZ9XIGgONeqm0sgB8B2O/i/tsAfAxAAIwBsNsbdXWythvs/pa3aqk2u7/7NgBbAUzTQl0ABgEoBDDMet0rrwF+ufcVaCPd6wAcUUqVKKWaAbwL4E6HZe4EsN56+T0AE8Q756PssDal1D+VUhetV3cBiPdCXZ2qzep/A3gJQKOG6noEwCqlVC1gOaZbQ7UpAAOslwcCqPJGYcpyApmz7SxyJ4C3lcUuAINEZKgWalNKfWX7W8K7r4HOPG4A8CSA9wF463nWmbpyAXyglCqzLu+12qjrAi104wCU212vsN7mdBmllBFAHYDBGqnN3hxYRiPe0GFt1k2QCUopb/aC68xjlgogVUT+JSK7RGSShmpbAuCn1sPjtsLyhq0FXX0u+oo3XwMdEpE4AHcD+J2va3GQCiBCRPJF5FsRecDXBZFrWj/3cq8kIj8FkA1gnK9rAQAR0QF4BcAsH5fiTBAsm5hzYBkVfSEiVyulzvmyKKv7AaxTSr0sItcDeEdERimlzL4uTOtEZDwsoXujr2ux898AnlFKmTXWrCUIQBaACQDCAOwUkV1KqWLflkXOBFroVgJIsLseb73N2TIVIhIEy2a/MxqpDSIyEcB/ARinlGryQl2dqa0/gFEA8q1vNkMAfCgidygXvZC9VBdgGaXtVkq1ADgmIsWwhPA3PVhXZ2ubA2ASACildopIKCwnqPf15r9OPRd9RUSuAbAWwK1KKW+8NjsrG8C71tdAFIDbRMSolNrs06osr4EzSqkGAA0i8gWAawEwdDUo0DYvfwMgRUSSRCQYlolSHzos8yGAB62XpwHYppTyxhlCOqxNRH4IS4/hO7y8X6bd2pRSdUqpKKVUolIqEZZ9bT0duB3WZbUZllEuRCQKlk1tJT1cV2drK4Nl9AERSQcQCuCUF2rryIcAHrDOYh4DoE4pdcLXRQGWGd8APgAwU2sjNaVUkt1r4D0A8zQQuADwNwA3ikiQiPQFMBrAQR/XRC4E1EhXKWUUkScAfALLLMM3lVIHRORFAHuUUh8CeAOWzXxHYJmcMENDtf0/AP0A/MX6abpMKXWHRmrzuk7W9QmA/xSRQgAmAL/wxuiok7UtAvAHEVkAy6SqWd74gCciG2H5IBJl3Z/8PIA+1rpfh2X/8m0AjgC4COChnq6pC7U9B8sci9XW14BReamLTidq84mO6lJKHRSRPAD7AJgBrFVK8Tz1GsXTQBIREXlJoG1eJiIi0iyGLhERkZcwdImIiLyEoUtEROQlDF0iIiIvYegSERF5CUOXyAesJ6bg64+ol+GLnshLRCTR2n/3bQD70fpUjETUC/DkGEReIiKJsJyi8gZrSz0i6mU40iXyrlIGLlHvxdAl8q4GXxdARL7D0CUiIvIShi4REZGXcCIVERGRl3CkS0RE5CUMXSIiIi9h6BIREXkJQ5eIiMhLGLpERERewtAlIiLyEoYuERGRl/x/8dgiZGLdABMAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAEGCAYAAAAgxE+CAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAA4uklEQVR4nO3deXxU9d0v8M93JmQhbAlJhmyYBBKyoLZNHkGrEASfGxXcAMW0KIKiRb1lsUrvUxW9txfspfZRlipFBW1BWrXUCsaqNIgVULRCIJAAgayQsIRAAllm8rt/zAxMJjNZJpOZM5PPm1dezHJy8s1kZj7zO+d3zleUUiAiIqLep/N2AURERH0FQ5eIiMhDGLpEREQewtAlIiLyEIYuERGRhwR4u4DeEBERoRISEtrc1tDQgNDQUO8U1AnW1n1arQtgba7Sam1arQvoWW0FlXVtrhvramC6WCfuqIuc88vQTUhIwJ49e9rclp+fj+zsbO8U1AnW1n1arQtgba7Sam1arQvoWW0Ji7e0uX5i/fyeF0Sd4uZlIiIiD2HoEhH1MfajXPIchi4REZGHMHSJiAjgGYE9gqFLRERorj7yrbdr6AsYukRERB7C0CUi6kM4icq7GLpEREQewtAlIiLyEIYuEVEfd3zZ7d4uoc9g6BIR9RHcn+t9DF0iIiIPYegSERF5CEOXiKgP4/5cz2LoEhH1Adyfqw0MXSIiIg9h6BIREXkIQ5eIqI/i/lzPC/DmDxeRNwFMBlCjlBrt4H4B8AqA2wBcBDBLKfWdZ6skIvJxSwbjWNCVqyYTMNK4wXv19GHeHumuA5DTwf23Aki2fM0F8HsP1ERE5D+WDIay9MoVMf+v1wMlgbneq6kP82roKqW+AHC2g0XuBPC2MtsFYIiIRHumOiIiH7dkMABz2FoD13pZ5Mr95DneHul2JhZAuc31CsttREREPkeUdbuDtwoQSQDwkZN9uh8BWKaU+tJy/XMAzyil9jhYdi7Mm6BhMBgy33333Tb319fXY8CAAe7/BdyAtXWfVusCWJurtFqbVusCOq9tfP6dkA6+XwHYnv03AMCECRO+VUplubdCsufViVRdUAkg3uZ6nOW2dpRSawCsAYCsrCyVnZ3d5v78/HzY36YVrK37tFoXwNpcpdXatFoX0IXa8gGlrmxatidAh9//7bffRgUEBKwFMBra3zKqFa0A9huNxoczMzNr7O/Ueuh+COAJEXkXwBgAdUqpE16uiYioTwgICFg7bNiwtMjIyFqdTufdzaI+orW1VU6dOpV+8uTJtQDusL/fq59cRGQjgJ0ARolIhYjMEZHHROQxyyJbAZQAOALgDwDmealUIiLfs6QOSuHyF9D2MpbUdbaG0ZGRkecZuF2n0+lUZGRkHcxbB9rx6khXKXV/J/crAI97qBwiIr+T1LwBRwJyodej7aFDnQcuAOgYuN1necwcDmq1vnmZiIhcZG1yMNK4ATBeuZ1novIe7hgnIiL3KNkeig33JWLVmDRsuC8RJdtDe7rKsrKygMmTJyfFx8ePzsjISBs/fvzIffv2BRUVFQUmJydnuKNse5cuXZLbb789afjw4aOvueaa1KKiokB3rZsjXSIi6rm8X8Zgz5sGGJt0gAJOF4WgJH8IsmZXI2dplSurbG1txR133DEyNzf3zEcffVQCADt37gypqqrql5iY2OzeX+CKV155JWLw4MHGsrKy/WvWrAlbuHBh3JYtW0rcsW6OdImI+pBe2bRcsj3UHLiN5sAFAKUExkYd9rxhcHXE+9FHHw0MCAhQTz/99Cnrbddff/2lnJycetvlioqKAjMzM0elp6enpaenp3366aehAFBaWtovKytrVGpqanpycnJGXl7eAKPRiKlTpyYkJydnpKSkpL/wwgtRDn7ukNmzZ58BgIceeqj2q6++Gtja2urKr9AOR7pERH7Io03rd62OMo9wHTA1C3atjkLS+GPdXe2+fftCrr322oudLRcTE2PcsWNHcf/+/VVBQUHQ/fffn7R///6Db775ZvjEiRPrXnrppZNGoxEXLlzQ7dy5s/+JEyf6HT58+AAAnD59Wm+/vurq6kDrSLpfv34YMGCAqbq6OiA6Otpov2x3MXSJiKhnao8HXx7h2lNKzPf3nubmZpkzZ85VhYWFITqdDqWlpUEAMHbs2IZHH300oaWlRTdt2rTaG2644VJqampTeXl50IMPPhg/ZcqUurvvvvt8b9Zmj5uXiYj8jEdHuQAQltAIEcepK6IQltDoymqvvvrqS3v37u3f2XK//vWvDVFRUS0HDx4sLCgoKGxpadEBwK233lr/xRdfFMXGxjbPnj07ceXKlUMjIyNN+/fvL5wwYcKF1157LXLGjBkJ9uszGAzNx44dCwSAlpYW1NfX6w0GQ49HuQBDl4jIr3QUuL12qNDYeTXQBzkOXX2gwth57U6H2BVTpky50NzcLMuXL4+w3rZ79+6QvLy8Niecrqur00dHR7fo9XqsXr16qMlkAgAUFxcHxsXFtSxatOj0Aw88cOq7777rf+LEiQCTyYRZs2adW7p0aWVBQUG7UL/99tvPvfnmm0MB4K233gq7/vrrL+h07olLhi4RkZ/obITbayPgpPENyJpdjYCg1ssjXhGFgKBWZM2pRtL4BldWq9Pp8OGHHx7dtm3boPj4+NEjR47MeOaZZ2JjY2NbbJebP39+zcaNG4eOGjUq/dChQ8EhISGtAPDJJ58MTEtLy0hLS0t///33w59++unq48eP97vxxhtHpaamps+cOTPpxRdfrLD/uT//+c9P19bWBgwfPnz0ihUrhi1fvrzdMq7iPl0iIuq5nKVVSMmpw67VUag9HoywhEaMnVfjauBaJSQktGzdutXh4TrWyVBXX311U3FxcaH19t///veVAPDkk0+eefLJJ8/Yf19hYeHBjn5m//791ccff+yWQ4TsMXSJiMg9ksY3uDJLuS/h5mUiIiIPYegSERF5CEOXiMhPdDY7mY0OvI+hS0TUBzBwtYGhS0Tk5xi42sHQJSLyEx4/E5Wd3Sd2hz7++eOJd26+M+3xzx9P3H1it0+29vv4448HpKenpwUEBGS+9dZbYe5cNw8ZIiKiHnvp65di/lL8F0OzqVmnoHCs7ljI7hO7h0xPmV79zHXP+FRrv6SkpOa33nrr+LJlywzuXjdHukRE1CO7T+wO/UvxXwxNpiadsjQ+UFDSZGrS/bn4zwZXR7zeau03atSo5jFjxlxy16kfbXGkS0TkxzyxP/ftwrejmk3NDhOqxdQibxe+HTUmeozPtPbrTQxdIiI/4M39uRUXKoKVk9Z+Ckoq6yvZ2s+Cm5eJiKhH4gbGNQoct/YTiIobEOdTrf16E0OXiMjHDfvn92j8z5grXxPdPv+nQw+kP1ATqA90GLr99P3UzPSZPtXarzcxdImIfNiwf34PKEveiZj/1+vReEu0x47PHRM9pmF6yvTqQH1gq3XEKxAVqA9svTfl3uox0WN8qrXf9u3b+xsMhmu2bt0atmDBgqtGjhzptkOTuE+XiMhHzVCWAZ81bO0uD/vn9zg54QceqeWZ656pyo7Prnu78O2oyvrK4NgBsY0PpD9Q42rgWnmjtd/48eMvVldX7+tJ3c4wdImIfJZHJ952akz0mAZXZin3Jdy8TETkj2xHv6QZDF0iIl+mHB+qQ9rE0CUi8lkmbxdA3cTQJSLyUe9KvXmka/0C2lz21CQq6jqvhq6I5IhIkYgcEZHFDu6fJSKnROR7y9fD3qiTiEirgj89AViOS7U9dIiBq01em70sInoAqwDcAqACwDci8qFSqtBu0U1KqSc8XiARkcbNyjMfjRP8eXWb273VP7dh567Qs+vXRzWXlwcHxsc3hj/4YE3o9WN7dMhQWVlZwLx584bv3bu3/6BBg0wREREtK1asKA8KClKTJ09Oth425E5LliwxvPPOOxF6vV4NHTrUuH79+uMpKSlu6WrkzZHudQCOKKVKlFLNAN4FcKcX6yEiIhed/L9LY8ofeyylfvv28OajR/vXb98eVv7YYykn/+/SGFfXaW3tN27cuAvl5eX7Dxw4cHDZsmWVVVVV/dxZu73MzMyL33///cHi4uLCu+66q3bBggVx7lq3N4/TjQVQbnO9AsAYB8tNFZFxAIoBLFBKlTtYBiIyF8BcADAYDMjPz29zf319fbvbtIK1dZ9W6wJYm6u0WptW6zJTANofGuTpeht27go9t2mTQTU1XRnIKSWqqUnObdpkGDhhQp0rI15nrf0Aczs/621FRUWBubm5iZcuXdIBwCuvvFJ2yy23NJSWlvabOnVqUn19vd5kMsmKFStKJ02aVH/fffcl7Nu3L1RE1E9+8pPTzz//fJvTVE6ZMuWC9fKNN95Yv2nTpqHdrd0ZrZ8c4+8ANiqlmkTkUQDrAdzsaEGl1BoAawAgKytLZWdnt7k/Pz8f9rdpBWvrPq3WBbA2V2m1Nq3WZe4q1D5wvbFp+ez69VGq2XFrP9XcLGfXr48KvX6sT7b2e/311yMnTZpU193anfFm6FYCiLe5Hme57TKllO3pu9YC+I0H6iIiom5oLi8Pdnq8sFLSXFHhk639Vq9eHb53797+r7/+epG7avXmPt1vACSLSKKIBAKYAeBD2wVEJNrm6h0AOjxfJhEReV5gfHwjxHFrP4iowDjfa+23efPmgcuXL4/eunXrkZCQELedgcRrI12llFFEngDwCcwnEH1TKXVARF4EsEcp9SGA/ykidwAwAjgLYJa36iUi0orPt43A2luuXDeZgEe3veq1WcvhDz5Y07Bz5xDV1NRue7cEBqrwBx90ubXfs88+K8uXL4946qmnTgPm1n61tbX6xMTEy7OJ6+rq9HFxcc16vR4rV65s09ovKSmpedGiRaebmprE0tqvLigoqHXWrFnnMjIyGmfOnJlk/3P/9a9/hTz55JNXbd269XBsbKzRldqd8eo+XaXUVgBb7W57zubyLwH80tN1ERFp1efbRrTp5KcUoNcDf5j0PwF4J3RDrx/bMOS++6rPbdpkUM3NAqUEIkoCA9WQ++6rdvWwIWtrv3nz5sW/8sorw4KCglRcXFzTihUr2kyonT9/fs3UqVNHvPvuu0NvvvnmOtvWfq+++uqwgIAA1b9/f9Of/vSnY8ePH+83Z86chNbWVgEAR639fvGLX8RfvHhRP3369BEAEBMT07xt27YjrvwO9rQ+kYqIiCw+3zYCgNNOfvh82whMvPmoh6syG/a/flk1cMKEurPr10c1V1QEB8bFueU4XW+09vvqq6+Ke1JzRxi6RETkFqHXj21wZZZyX8JzLxMR+QF28vMNDF0iIh/CTn6+jaFLRETkIQxdIiIfMfHmow47+dneT9rG0CUi8hEJi7fgkc9ebdfJD2Dg+grOXiYi8jGPbnu1zXVvnRTDXsWhs6Hff14edf70peBBESGNP5gYXxOXGu5zrf1+85vfRK5duzZSp9MhNDTUtGbNmtLMzEyXzqplj6FLREQ9tuPPxTEHdlQZTC2tOgCoPXkxpOJQ7ZCMm2Kqb7o3pcqVdVpb++Xm5p756KOPSgBg586dIVVVVf1sz0jlbg8//PAZa2ejP/3pT4Pnz58fv2PHjsPuWDc3LxMR+QBzV6H2tDDKrTh0NtQ2cAEACmJqadUd+KLKUHHobKgr63XW2i8nJ6fedrmioqLAzMzMUenp6Wnp6elpn376aSgAlJaW9svKyhqVmpqanpycnJGXlzfAaDRi6tSpCcnJyRkpKSnpL7zwQpT9zw0PD2+1Xq6vr9eLG4/H4kiXiIh65PvPy6PaBK4Nk6lVvv+8PCouNdynWvstXbo0cvXq1YaWlhbdp59+6hddhoiIyA+cP33Jees+Benwfjdobm6W3NzchJSUlPTp06ePOHr0aDBgbu23cePGiIULF8Z8/fXXIWFhYa22rf3ee++9QWFhYSZH6/zlL395qry8fP+SJUsqnn/++WhHy7iCoUtEpHHONi2vy3Fpq63bDYoIaYTASWs/qEERIT7X2s/qkUceOfvpp58OcaV+Rxi6RETUIz+YGF+jD9A5DF29Xqd+MDHe5dZ+zc3Nsnz58gjrbbt37w7Jy8sbYLtcXV2dPjo6ukWv12P16tVtWvvFxcW1LFq06PQDDzxwytLaL8BkMmHWrFnnli5dWllQUNAu1AsKCoKslzdt2jT4qquuanKlfke4T5eISMOWLFmCB4OuXDeZgD8a/8N7BTkQlxrekHFTTPWBL6oMJlOrQEEgUHq9TmWMi6l29bAhb7X2e/nll6N27NgxKCAgQA0ePNi4bt06tzVxYOgSEWnUkiVLLl+27Z37gO4bvPjiEuTn53upsvZuujelKvGaiDp3H6frjdZ+b731VnlH9/cEQ5eISIPsA9f2f+v92dnZHq2pM3Gp4Q2uzFLuS7hPl4iIyEMYukREPoa9c30XQ5eISKPYO9f/MHSJiIg8hKFLRKRBS5Yscdg713rZdqIV+Q6GLhGRBiUs3oK3m/+jXe9cEe0Gbtn+vaF/femFxHWLfpb215deSCzbv7fHp8wqKysLmDx5clJ8fPzojIyMtPHjx4/ct29fUFFRUWBycnKGO+p2Zt26dUNEJPOLL77o9KxYXcVDhoiINOyPxv8AjFeua6GrkCP/XPeHmL2ffWwwGVt0UApnKitCSgv2Drl20q3VE2Y94lOt/QCgtrZWt3LlSsM111zTo+OM7XGkS0REPVK2f2/o3s8+NphamnU228LF1NKs2/vZVoOrI15vtfYDgEWLFsU+9dRTJ4OCgtw6nY0jXSIijdFy71xHvt2yOcpkbHHc2s9olG+3bI4aPvpan2nt9+WXX/avrKwMnDFjRt3LL788rLt1d4ShS0REPVJXczLY6fFNSkldTXWvt/abM2fOVYWFhSE6nQ6lpaVBgLm136OPPprQ0tKimzZtWu0NN9xwyba135QpU+ruvvvu87brMplMWLhwYfw777zTK2fW4uZlIiLqkcFRwxoh4qS1n6jBUQafae137tw5/eHDh4NvvvnmUbGxsVfv3bs3dNq0aSPdNZmKoUtEpCG+tmkZADJvv6tGH9DPcWu/gACVeftdPtPab+jQoaba2tq9lZWVBZWVlQXXXnttw3vvvXdk3LhxnW7m7gq/37xc//UJnPvgCEZAh4q8HQ6XiVt2k4erIiJqr2LxDuzAwMvXTTAhG255r+9Vw0df23DtpFur93621WAyGgVKCUSUPiBAXTvpturho6/1qdZ+vcmroSsiOQBeAaAHsFYptczu/iAAbwPIBHAGwH1KqeNdXX/N63vRfMy8uV7g/GSlFYt3MHiJyKsqFl8ZFAgECgp66PEFBmAc6jv4Tm2YMOuRqhFZ19V9u2VzVF1NdfDgKENj5u131bgauFbeaO1n6+uvvy7qftXOeS10RUQPYBWAWwBUAPhGRD5UShXaLDYHQK1SaqSIzADwEoD7urL++q9PXA7czigoBi8ReY194Nr+DwBfYpDHa3LF8NHXNrgyS7kv8eZI9zoAR5RSJQAgIu8CuBOAbejeCWCJ5fJ7AFaKiCjV+WnAL2wr61YxCsrpvhRbWt6vQkRE2ubN0I0FYLtdvgLAGGfLKKWMIlIHYCiA0/YrE5G5AOYCgMFgQMv5Jug62KTc5nstm3K6wlEwr8vp2ZnO6uvrkZ+f36N19Bat1qbVugDW5iqt1uaJukZA53QXmPX9yVENWn3MyDm/mUillFoDYA0AZGVlqX6DgmA617WzhHU1cJ2Zldd+l0V3RsT5+fnIzs7uUQ29Rau1abUugLW5Squ1eaKuirwdUFAdBq+jGrT6mJFz3gzdSgDxNtfjLLc5WqZCRAIADIZ5QlWnBt48HOc+ONLlYkwwdXnZrrAfEXOzNBERefM43W8AJItIoogEApgB4EO7ZT4E8KDl8jQA27qyPxcABlwXjcDEjicfKJt/vT0tP2HxlstfRES24pbd1Ob9CECby5zk6T+8NtK17KN9AsAnMB8y9KZS6oCIvAhgj1LqQwBvAHhHRI4AOAtzMHdZ1KPXXj5O19GmG+v1+GXjcLyD9bg7KG3XxxEwEVnfE/LRH3roL4etQHwqcBuPnAut/7Iyyni2MTggPLhxwI2xNcEjh/TokKGysrKAefPmDd+7d2//QYMGmSIiIlpWrFhRHhQUpCZPnpxsPWzInV599dWhzz//fJzBYGgBgLlz59YsXLiw3VwiV3h1n65SaiuArXa3PWdzuRHA9J78jAHXRWPAddE92vfhKBjdFcSX15O3hQFM1MfZb3HzpfeEc38/GlO/+4QBRqUDAOOpiyGNR2qHDBgTXT1kygifa+03ZcqU2rfffrt7h8F0gdPQFZFVADYopf7l7h/qD3ojiK3f70svNCKixiPnQm0DFwCgIDAqqd990hCcNrTOlRGvs9Z+gLmdn/W2oqKiwNzc3MRLly7pAOCVV14pu+WWWxpKS0v7TZ06Nam+vl5vMplkxYoVpZMmTaq/7777Evbt2xcqIuonP/nJ6eeff96l01S6oqORbjGA5SISDeDPADYqpf7tmbJ8k31YuhrC3PxM1Hf44rmW7dV/WRnVJnBtmVql/svKqOCRQ3ymtR8AfPzxx0NSUlIGJCUlNa5cubJ85MiRLd2t3xGnE6mUUq8opa4HMB7mGcNvisghEXleRFLc8cP93fFlt1/+chUnXxGR1hnPNjpv3acgHd7vBs3NzZKbm5uQkpKSPn369BFHjx4NBsyt/TZu3BixcOHCmK+//jokLCys1ba133vvvTcoLCys3aEr995777mysrKC4uLiwokTJ57/6U9/muiuWjudvayUKlVKvaSU+iGA+wHcBaDL560ks54GMMOXiLQqIDy4EeLkhAcCFRAe5DOt/QBg2LBhppCQEAUACxYsOH3gwAG3tPUDuhC6IhIgIlNE5E8APgZQBOAedxXQF/UkgHnoEZH/8IdNywAw4MbYGuid9NPV69SAG+N8prUfAJSWlvazXt6wYcOQpKQklz40ONLRRKpbYB7Z3gbgawDvApirlOrR9G9q6/iy25Gfn+/wrFad4cQrItKC4JFDGgaMia6u333SAFOrWI7QVNDr1IAxw6pdPWzIW639fvOb30R98sknQ/R6vRoyZIhx3bp1x12p35GOJlL9EsAGAIuUUrXu+oHkmG1wdncU+9J9k6EH2hyFvGjTR+4pjIh6xW/vm4wnbK6bAPw+8WfeKqfHhkwZURWcNrTO3cfpeqO136pVqyrR/gyJbuE0dJVSN/fGD6TOWQO4K+H7+LHftwlbBXP4/va+yQxeIo367X2TL+8AFZhft3qYX8+/8OHXbfDIIQ2uzFLuS7x5GkjqRGf7fX9mCVzrF2z+VzC/sIlIW6yvS/vXrfWLr1v/5jddhvyZs03PDg8us9Gz3klE1FusW6So7+FI18fYj36dvXCtt3OmM5H2dPa6Jf/F0PVRx5fdfnlfkCP2tzN8ibSDW6H6Loauj+vok7GjDsEMXyLvMnq7APIqhq4PczQ7Wdl8dXT4AYOXyDt+n/izNq9ToO3I15ePOigpKQndsGFD4qpVq9I2bNiQWFJSEtrTdZaVlQVMnjw5KT4+fnRGRkba+PHjR+7bty+oqKgoMDk5OcMddTuydu3asBEjRmSMHDkyY8qUKW47DSQnUvm4RZs+ajPbUWD+JN2V4/14cg0i71iV+DP87NjvoYf/BG5eXl7Mnj17DEajUQcAp0+fDikpKRmSlZVVnZOT41Ot/QoKCoJ++9vfRu/atetQZGSkqbKy0m1ZydD1A45eqM+g66NZhi+RZ9i+Jm0/GPv6a6+kpCTUNnABQCklRqNR9uzZY0hJSalLSkrymdZ+q1atinzkkUdqIiMjTQAQGxvrtr0CDF0/1p2TbFiX8/UXPxF53q5du6JsA9eWyWSSXbt2RSUlJflMa78jR44EAcCPfvSjVJPJhGeffbZq2rRp57tbvyMM3T6gO+HLUS9R7/DneRS1tbVOW/cppaSj+92hublZ5syZc1VhYWGITqdDaWlpEGBu7ffoo48mtLS06KZNm1Z7ww03XLJt7TdlypS6u+++u12YmkwmOXr0aNDOnTuLjh071i87Ozs1Ozv7QEREhKP5qd3CiVR9SHeC1J/fIIi0xB8+4IaFhTWKOO4yJCIqLCzMp1r7RUdHN0+ePPlcUFCQSk1NbU5MTGw8cOBAkCu/gz2Gbh/TnZaCCYu3uNT9iIj6lrFjx9bo9XqHoavX69XYsWN9qrXfPffcc2779u0DAeDEiRMBx44dCx41alSTK7+DPYZuH9Xd8CUi1/lL31xnkpKSGrKysqoDAgJarSNeEVEBAQGtWVlZ1a5MogKutPbbtm3boPj4+NEjR47MeOaZZ2JjY2NbbJebP39+zcaNG4eOGjUq/dChQ8G2rf3S0tIy0tLS0t9///3wp59+uvr48eP9brzxxlGpqanpM2fOTHLU2u+ee+45Hx4ebhwxYkTG+PHjU1588cXyYcOG9XjTMsB9un3e8WW3dylUX3nsH9BDD7E7Hcfjr7EZFVFHVj22DU/hyi5NE0z43ZCWDr7DN+Xk5FSlpKTU7dq1K6q2tjY4LCyscezYsTWuBq6VN1r76XQ6rF27tgJAu0DuKYYudTrRatG5oHZha7XqsW0MXiInVj22DcpyJK65q7uCHnosOqfDb4e4ZWulpiQlJTW4Mku5L+HmZbrM0aauBef6QWz+ObLqsW29XRqRz7G+LmxfO7avpV+cC/FmeeQlDF1qw35fr77TBoJERNRVDF1yqG37wI4bjin2TCHqls5eU+S/GLrklLl9oHQYqtb7OMOZqD1+ICV7DF3qVGefyk2WJoIMXqIrjGziRw4wdKlDGTMcP0WUzT/bwx/Yr5fI7HdDWtq8ToC2I19/nPV/9uxXod/vfThx167/kfb93ocTz579yidb+82ZMyc+NTU1PTU1NT0hIWH0wIEDf+CudfOQIerU46/d3GaGsvWNo6PjDdk8gQj47ZAmLDjXD3ro/T5wi4v/T0xl1QZDa2uzDlBouHg0pLb2qyGxMbnVKSm/8qnWfm+88Ua59fKvf/3rqO+//77TU1F2lVdCV0TCAWwCkADgOIB7lVK1DpYzASiwXC1TSt3hqRqpLUdvEp2NaBm81FfZvjbMH0zNH0799fVw9uxXoebAbbLZNKaktbVJKqs2GCIibq4LD7/BZ1r72XrvvffCn3vuOZc+NDjirZHuYgCfK6WWichiy/VnHCx3SSn1A49WRl3WlbNZsWsRkf8rK38zyjzCba+1tVnKyt+MCg+/wWda+1kVFxcHVlRUBE6ZMsUtbf0A7+3TvRPAesvl9QDu8lId1EM8fzNRW33xud54qTwYTmdqK2m8VNHrrf1yc3MTUlJS0qdPnz7i6NGjwYC5td/GjRsjFi5cGPP111+HhIWFtdq29nvvvfcGhYWFOT2n8vr168Nvu+222oAA941PvTXSNSilTlgunwRgcLJcsIjsAWAEsEwptdnZCkVkLoC5AGAwGJCfn9/m/vr6+na3aYWv17YuxzxXorOORAmLt1xe1hN1eQtrc41Wa3NXXetyQt3++2nlMQsOiW9suHg0BFAODnUQFRwS73Jrv82bN4d1tpy1td/7779/rLW1FSEhIZnAldZ+77///uDZs2cnPvHEE9VPPPHEmf379xf+9a9/HfTaa69Fbtq0Kfwvf/nLcUfr/eCDD8JfffXVUldqd6bXQldEPgMwzMFd/2V7RSmlnPVhBHCVUqpSRJIAbBORAqXUUUcLKqXWAFgDAFlZWSo7O7vN/fn5+bC/TSv8pbbj2Z1/yrcGc083N/vLY+ZprK37ulNXR8//3vjdtPKYDY+fXVNb+9WQ1tamdqGr0wWq4fEPudza79lnn5Xly5dHPPXUU6cBc2u/2tpave1Eqrq6On1cXFyzXq/HypUr27T2S0pKal60aNHppqYmsbT2qwsKCmqdNWvWuYyMjMaZM2cmOfrZ//73v4PPnz+vnzhxolv7m/ba5mWl1CSl1GgHX38DUC0i0QBg+d/hH0QpVWn5vwRAPoAf9la95B7c3EzUnr/PaQgPv6EhNia3WqcLagWsgyhROl1Qa2xMbrUrk6gA77X2A4B33nkn/M477zyr07k3Jr21eflDAA8CWGb5/2/2C4hIGICLSqkmEYkA8GMAv/FoleSSzroWWXF2M/mTvv5BMiXlV1URETfXlZW/GdV4qSI4OCSucXj87BpXA9fKG639AODll19224xlW94K3WUA/iwicwCUArgXAEQkC8BjSqmHAaQBeF1EWmEekS9TShU6WyFpT2ezm/+++SkUbn4KANqc8yrtUKevByJNOZiahq02100Apty1HID/j3JthYff0ODKLOW+xCuzl5VSZ5RSE5VSyZbN0Gctt++xBC6UUl8ppa5WSl1r+f8Nb9RKPePsDWfL5qcu9y+y3wl0MDWtV2sicifb56v1uayH+TlOZI+ngaReZ98u8O+bn4IAl78cYfCSL3AUuLbP7Y8ZvGSHoUseYw1eduglor6KoUseZQ3ezrqJsiEa+Tp2zCVHGLrkcV0N3L4+G5R8Az8gUncwdEmTrOdlY/CSlrFjbls7ai+EztxXkjhu98G0mftKEnfUXvDJ1n6HDx8OHDNmTEpaWlp6SkpK+qZNmwa7a90MXfI4Z4cEKZsv6+EWAIOXtGvKXcvbPG+BtiPfvnT427OHK2J+uq8k5bMz58OLLzb1/+zM+bCf7itJefZwRYyr67S29hs3btyF8vLy/QcOHDi4bNmyyqqqqn7urN3ec889F33PPffUHjx4sHDjxo0lCxcuHO6udTN0ySvs34ysb1QmALfbBK5VwuItDF/SFOvz8fa7ll/eMtNXA3dH7YXQt6vOGJpalc7mw4c0tSrd21VnDK6OeJ219svJyam3Xa6oqCgwMzNzVHp6elp6enrap59+GgoApaWl/bKyskalpqamJycnZ+Tl5Q0wGo2YOnVqQnJyckZKSkr6Cy+8EGX/c0UE58+f1wNAbW2tPioqynHjcBewiT15jaM3JZ7FinyR7ZaZvvj8XFN+Kqq5VTkcxDW3KllTfirqprCBPtPab+nSpVW33HJL8tq1a6MuXbqk27JlS3F3a3eGI13SlK68YXHES97G52BbpZeagp029gOk7FKTT7X2e+utt8Lvv//+M9XV1fs++OCDw7NmzUq0NlHoKYYuaU5XgrezNoJE3tAXR7kAcFVIUKM4mcgtgBoeEuRya7+9e/f272w5a2u/gwcPFhYUFBS2tLTogCut/WJjY5tnz56duHLlyqGRkZGm/fv3F06YMOHCa6+9FjljxowE+/X98Y9/jJg5c+ZZAJg0aVJDU1OT7uTJk27ZMszQJU3q6oiXIw7yND7n2psbH1kTqHPcojVQJ2pufKTLrf2am5tl+fLlEdbbdu/eHZKXlzfAdrm6ujp9dHR0i16vx+rVq9u09ouLi2tZtGjR6QceeOCUpbVfgMlkwqxZs84tXbq0sqCgoF2ox8TENG/dunUQAHz33XfBzc3NEh0d7ZbJ6gxd0iz700c6wzdB0oK+OsoFgJvCBjY8EDO0OkgnrdYRrwAqSCetD8QMrb4pbKBPtfb73e9+V75u3brIUaNGpefm5ia99tprx93V4o8TqUjzOutWBHCCFXkGP+A597+T46r+M2Jw3ZryU1Fll5qCh4cENc6Nj6xxNXCtvNHaLzMzs/G777471JO6nWHokk/oSvCmvXE1Auye0QUPFvRiVdRXXL3+agDAgFTzdZMJuHR42eX7+YHP7KawgQ2uzFLuS7h5mXxGR29soaMWQ68HlN1eJeubJZGrnix9EoD5uSWWc5jq9ebnHFF3MXTJpzjazxuSvBgiuPxlj8FLrrJ97lifW7bPtZDkxRzlUrcwdMknrcu5coIbPXsFkpfwuUfdxdAln2U7wnA0wiXqTc62rBB1hKFLPq0rm/aU4qxTItIGhi75vI5GG7YTqxi81F1KtZ+cR87968jp0Nnrvkm85eXtabPXfZP4ryOnfbK1X3FxceD111+fkpKSkn7dddeNOnr0qNu6GjF0ye/ZnjKVwUvd4abT7fYJL/79QMzsdd+k/PNQTfjhmvr+/yyqCZu97puUF/9+wOda+/385z+Py83NPVNcXFz4q1/9qmrRokVx7lo3Q5d8nqNjca0jFKXaHk8JMHipaxIWb8Glw8vaPJeAtiNfHgdu9q8jp0P/tLvM0GRsvdLaT0GajK26P+0uM7g64vVWa7/Dhw+H3HrrrecBYPLkyRc+++yzIa7U7whDl/yCozc/kwloKFrmYGkGL3VdQ9GyyyNe22N1GbhXvPHlsahmY6vj1n6mVnnjy2Ptgq0rutvar7Cw8OCmTZtKFixYMBwArK39Dh06VHjw4MEDY8aMuWjb2q+4uLjw8ccfb3fGqrS0tIsbN24MA4B33nlnSENDg+7kyZNumavOM1KR33D0JthRuFrv43GWZM/+ecOzT3Ws/OxF5639FKT87MVeb+03Z86cqwoLC0N0Oh1KS0uDAHNrv0cffTShpaVFN23atNobbrjhkm1rvylTptTdfffd5+3Xt2LFioq5c+cOT0tLixg7duyFqKiolgD70925iCNd8mtsmEDd1dHzgYHrWHx4/0YRJ639BGp4eH+fau2XkJDQ8o9//OPowYMHC//7v/+7EgAiIiLcsoefoUt+j8FL1Lvm3JhYE6jXOW7tp9ep2Tcm+lRrP+syAPCrX/0q+v777z/tSv2OMHSpT2DwUldwlOuaH4+MaPjJmOHVQQG6VuuIVwQqKEDX+pMxw6t/PDLCp1r75eXlDUxKShqdkJAwuqamJmDp0qUnXKnfEe7TpT6jqy0CrcsSWfH50LnnpmRUTUwz1L3x5bGo8rMXg+PD+zfOuTGxxtXAtfJGa7+HHnqo9qGHHqrtSd3OMHSpT7G+ebI/L9njlo6e+/HIiIYfj4xga78OMHSpT+ps1HskIBfqebuzXS2p6/3CyLOWDAZgPhToWJD5JpMJGGnccHkR2+YaRD3llX26IjJdRA6ISKuIZHWwXI6IFInIERFh80pyK2cj2ZLA3DbdYy6fDMHyBk1+wubvadu8QK83PweIeoO3JlLtB3APgC+cLSAiegCrANwKIB3A/SKS7pnyqK+wD94jAbntevO2He0yeP2Ck7+j7d/+SEAudzGQ23ll87JS6iAASMd9sa4DcEQpVWJZ9l0AdwIo7OibiLrLdj9vR/1Rbc9GRP7PTedCIGpDy0+rWADlNtcrAIxxtrCIzAUwFwAMBgPy8/Pb3F9fX9/uNq1gbd3XG3WtywkF/uk8WEXMwbu9k5+r1ccMYG1W4wF09PlJBFAw/635mJE79VroishnAIY5uOu/lFJ/c/fPU0qtAbAGALKyslR2dnab+/Pz82F/m1awtu7rtbrynY9orft2Z+WZj4BwtulRq48ZwNqu/LDOt1wIgOzsbD5m3VGyPRS7Vkeh9ngwwhIaMXZeDZLG9+iQobKysoB58+YN37t3b/9BgwaZIiIiWlasWFEeFBSkJk+enGw9bMidPv744wGLFi2KLy4u7v+HP/yhxPbwoRUrVgxdvnx5NAA89dRTJxwdktSRXgtdpdSkHq6iEkC8zfU4y21EvaqjN2L7NoHc5+ebWoxAgFtOX0+X5f0yBnveNMDYpAMUcLooBCX5Q5A1uxo5S6tcWaW1tV9ubu6Zjz76qAQAdu7cGVJVVdUvMTGx2b2/wBVJSUnNb7311vFly5YZbG+vrq7Wv/TSSzHffvttoU6nww9/+MP0GTNmnIuMjOzyKSK1fEaqbwAki0iiiAQCmAHgQy/XRP7OwWFBtq3dbA8lAXhspy9KWLwFycYN7Vr2tcNDxLquZHuoOXAbzYELAEoJjI067HnDgJLtPtXab9SoUc1jxoy5pNO1jcjNmzcPHjdu3HmDwWCKjIw0jRs37vwHH3zQrdmVXtmnKyJ3A1gBIBLAFhH5Xin1P0QkBsBapdRtSimjiDwB4BMAegBvKqXcvhmBqJ0lde1mt9ofu2mLZ7HyTUnNG3A4IBcB+ivBe3krBwO3e3atjjKPcB0wNQt2rY5C0vhunzSju639+vfvrwoKCoLuv//+pP379x+0tvZ76aWXThqNRly4cEFn29oPAE6fPt3lbR6VlZX94uLiLo+wY2NjmysrK/t153fy1uzlvwL4q4PbqwDcZnN9K4CtHiyNyMzmTVcAjOzCiJabm7XPfstEsnEDYDRf5t+uB2qPB8NxkyHziLf2uE+19utNWt68TKQZx5fdzqYJPo7NDHpRWEIjRBynrohCWIJPtfZzJjY2tqWioiLQer2ysjLQvvlCZxi6RN3QlTfnWXkNDF+N4d+jl42dVwN9kOPQ1QcqjJ3nU639nLnrrrvqtm/fPujUqVP6U6dO6bdv3z7orrvu6ta+CIYuUTd1dVTEN3pt6OzvwFGuGySNb0DW7GoEBLVeHvGKKAQEtSJrTrWrhw15q7Xf9u3b+xsMhmu2bt0atmDBgqtGjhyZAQAGg8H0i1/8oiozMzMtMzMz7emnn64yGAzdam6v5ZNjEGlWd7oV2S5P2sK/ixvlLK1CSk6du4/T9UZrv/Hjx1+srq7e5+i++fPnn5k/f363js21xdAl6oGu9OgFOMnKW7gf18OSxje4Mku5L+HmZaIe4uZmbWLgkhYxdIncoDuzmxm+vY+Psdu0tra2ss1HN1kes1ZH9zF0idyIo17v48Qpt9p/6tSpwQzermttbZVTp04NhrmFbTvcp0vkZutyQpGdnc1JVl7AwHUvo9H48MmTJ9eePHlyNDhI66pWAPuNRuPDju5k6BL1ku5MsrIuT65j4LpfZmZmDYA7vF2HP+EnF6Je1J03em5ydh0Dl3wFQ5eol3V1khXAiVauYOCSL2HoEnkIR73ux8AlX8N9ukQe1NUzWdkuw+BoryuPHx830iKOdIm8gJucXcfAJV/G0CXyou5ucu7r4cvAJV/H0CXysu6MeoG+G74MXPIHDF0ijehuYPSV8O3q78nAJV/AiVREGtKdiVZW/jrhqquPgb/93uTfONIl0qDubnIG/Gvky8Alf8WRLpGG9bWRb3d+T1/8/YgYukQ+oCfha/v9WtXdEbrWfx8iZxi6RD7ElfC1XX5dTqjba+oJhi31NQxdIh/kavjOymsA8ry7+dmV/c4MW/IXDF0iH+Zq+Dr6nt4Mtp5M8GLgkj9h6BL5gZ6Er5Wj73Ul8Nw1g5phS/6IoUvkR2yDyh3h541DkBi25M8YukR+yh2jX09alxOK7Oxsb5dB1KsYukR+zt2jX3eyrS0/P997hRB5CEOXqA+xH01y8zGRZ3kldEVkOoAlANIAXKeU2uNkueMALgAwATAqpbI8VSNRX+AoAN0ZxAxYora8NdLdD+AeAK93YdkJSqnTvVwPEVkwKIl6j1dCVyl1EABExBs/noiIyCu0vk9XAfiHiCgAryul1jhbUETmAphruVovIkV2i0QA0OqImbV1n1brAlibq7Ram1brAtxb21VuWg91oNdCV0Q+AzDMwV3/pZT6WxdXc6NSqlJEogB8KiKHlFJfOFrQEsgdhfIere4TZm3dp9W6ANbmKq3WptW6AG3XRo71WugqpSa5YR2Vlv9rROSvAK4D4DB0iYiItE6zTexFJFREBlovA/hPmCdgERER+SSvhK6I3C0iFQCuB7BFRD6x3B4jIlstixkAfCkiewF8DWCLUiqvBz/W6aZnDWBt3afVugDW5iqt1qbVugBt10YOiFLK2zUQERH1CZrdvExERORvGLpEREQe4nehKyI5IlIkIkdEZLGD+4NEZJPl/t0ikqCh2haKSKGI7BORz0XEY8fNdVabzXJTRUSJiEcOU+hKXSJyr+VxOyAiGzxRV1dqE5HhIvJPEfm35W96m4fqelNEakTE4cRDMXvVUvc+EfmRJ+rqYm0/sdRUICJfici1WqnNZrn/EBGjiEzTSl0iki0i31teA9s9URe5SCnlN18A9ACOAkgCEAhgL4B0u2XmAXjNcnkGgE0aqm0CgP6Wyz/TUm2W5QbCfMjWLgBZWqgLQDKAfwMIs1yP0spjBvMkl59ZLqcDOO6h2sYB+BGA/U7uvw3AxwAEwFgAuz1RVxdru8Hmb3mrlmqz+btvA7AVwDQt1AVgCIBCAMMt1z3yGuCXa1/+NtK9DsARpVSJUqoZwLsA7rRb5k4A6y2X3wMwUTxzPspOa1NK/VMpddFydReAOA/U1aXaLP43gJcANGqorkcArFJK1QLmY7o1VJsCMMhyeTCAKk8UpswnkDnbwSJ3Anhbme0CMEREorVQm1LqK+vfEp59DXTlcQOAJwG8D8BTz7Ou1JUL4AOlVJlleY/VRt3nb6EbC6Dc5nqF5TaHyyiljADqAAzVSG225sA8GvGETmuzbIKMV0p5shdcVx6zFAApIvIvEdklIjkaqm0JgJ9aDo/bCvMbthZ097noLZ58DXRKRGIB3A3g996uxU4KgDARyReRb0XkAW8XRM5p/dzLfZKI/BRAFoDx3q4FAEREB+BlALO8XIojATBvYs6GeVT0hYhcrZQ6582iLO4HsE4p9VsRuR7AOyIyWinV6u3CtE5EJsAcujd6uxYb/w3gGaVUq8aatQQAyAQwEUAIgJ0iskspVezdssgRfwvdSgDxNtfjLLc5WqZCRAJg3ux3RiO1QUQmAfgvAOOVUk0eqKsrtQ0EMBpAvuXNZhiAD0XkDuWkF7KH6gLMo7TdSqkWAMdEpBjmEP6mF+vqam1zAOQAgFJqp4gEw3yCem9v/uvSc9FbROQaAGsB3KqU8sRrs6uyALxreQ1EALhNRIxKqc1ercr8GjijlGoA0CAiXwC4FgBDV4P8bfPyNwCSRSRRRAJhnij1od0yHwJ40HJ5GoBtSilPnCGk09pE5Icw9xi+w8P7ZTqsTSlVp5SKUEolKKUSYN7X1tuB22ldFpthHuVCRCJg3tRW0st1dbW2MphHHxCRNADBAE55oLbOfAjgAcss5rEA6pRSJ7xdFGCe8Q3gAwAztTZSU0ol2rwG3gMwTwOBCwB/A3CjiASISH8AYwAc9HJN5IRfjXSVUkYReQLAJzDPMnxTKXVARF4EsEcp9SGAN2DezHcE5skJMzRU2/8DMADAXyyfpsuUUndopDaP62JdnwD4TxEpBGAC8AtPjI66WNsiAH8QkQUwT6qa5YkPeCKyEeYPIhGW/cnPA+hnqfs1mPcv3wbgCICLAB7q7Zq6UdtzMM+xWG15DRiVh7rodKE2r+isLqXUQRHJA7APQCuAtUopnqdeo3gaSCIiIg/xt83LREREmsXQJSIi8hCGLhERkYcwdImIiDyEoUtEROQhDF0iIiIPYegSeYHlxBR8/RH1MXzRE3mIiCRY+u++DWA/2p6KkYj6AJ4cg8hDRCQB5lNU3mBpqUdEfQxHukSeVcrAJeq7GLpEntXg7QKIyHsYukRERB7C0CUiIvIQTqQiIiLyEI50iYiIPIShS0RE5CEMXSIiIg9h6BIREXkIQ5eIiMhDGLpEREQewtAlIiLykP8PO6McxqR28R4AAAAASUVORK5CYII=\n", "text/plain": [ "
" ] @@ -159,7 +175,7 @@ "metadata": {}, "outputs": [], "source": [ - "model = symdet.DenseModel(n_layers=7,\n", + "model = symsuite.DenseModel(n_layers=7,\n", " units=80,\n", " epochs=10,\n", " batch_size=64,\n", @@ -341,7 +357,7 @@ } ], "source": [ - "sym_detector = symdet.GroupDetection(model, double_well_potential.clustered_data)\n", + "sym_detector = symsuite.GroupDetection(model, double_well_potential.clustered_data)\n", "point_cloud = sym_detector.run_symmetry_detection(plot=True, save=True)" ] }, @@ -378,6 +394,527 @@ "source": [ "We see that the clustering has worked relatively well. This is where some caution is advised in terms of simply parsing this data along to the generator detection. If you see that one of these groups should likely not be in the set then it should be parsed along to the generator extraction stage. Alternatively, you can use a different approach for identifying the sets in the symmetry representation. In the next update we will parse this data to the generator extraction algorithm in order to get the generators of this detected symmetry group." ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [], + "source": [ + "from flax import linen as nn\n", + "import jax\n", + "import jax.numpy as jnp\n", + "import numpy as np\n", + "\n", + "class ProductionModule(nn.Module):\n", + " \"\"\"\n", + " Simple CNN module.\n", + " \"\"\"\n", + "\n", + " @nn.compact\n", + " def __call__(self, x):\n", + " x = nn.Conv(features=128, kernel_size=(3, 3))(x)\n", + " x = nn.relu(x)\n", + " x = nn.max_pool(x, window_shape=(3, 3), strides=(2, 2))\n", + " x = nn.Conv(features=128, kernel_size=(3, 3))(x)\n", + " x = nn.relu(x)\n", + " x = nn.max_pool(x, window_shape=(3, 3), strides=(2, 2))\n", + " x = x.reshape((x.shape[0], -1)) # flatten\n", + " x = nn.Dense(features=300)(x)\n", + " x = nn.relu(x)\n", + " x = nn.Dense(10)(x)\n", + "\n", + " return x" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "model = ProductionModule()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "key = jax.random.PRNGKey(452)\n", + "\n", + "data = model.init(key, jnp.ones([1, 28, 28, 1]))" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "FrozenDict({\n", + " Conv_0: {\n", + " kernel: DeviceArray([[[[ 0.37528017, 0.13753012, 0.35983738, ...,\n", + " 0.31934947, -0.10718577, 0.06134491]],\n", + " \n", + " [[-0.03669485, 0.14023171, 0.17660904, ...,\n", + " 0.17953686, -0.09349817, 0.5601775 ]],\n", + " \n", + " [[ 0.00360224, -0.23205283, -0.4305723 , ...,\n", + " -0.55263746, 0.08736772, -0.33638862]]],\n", + " \n", + " \n", + " [[[ 0.3732802 , -0.3748587 , 0.18389685, ...,\n", + " -0.31333193, 0.2105282 , 0.02409167]],\n", + " \n", + " [[-0.46691698, 0.49650607, -0.5371514 , ...,\n", + " 0.01834795, 0.56708217, -0.34570318]],\n", + " \n", + " [[ 0.5792693 , -0.35017362, 0.5757499 , ...,\n", + " -0.41792697, -0.20277935, 0.1124992 ]]],\n", + " \n", + " \n", + " [[[-0.5241861 , -0.08293429, -0.15216689, ...,\n", + " 0.48141024, -0.00571131, -0.00457093]],\n", + " \n", + " [[ 0.15416023, 0.04572708, -0.05879792, ...,\n", + " 0.57276475, 0.09112789, 0.07591753]],\n", + " \n", + " [[-0.21100795, -0.17283523, 0.07323457, ...,\n", + " 0.02905115, -0.47458482, 0.04670273]]]], dtype=float32),\n", + " bias: DeviceArray([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32),\n", + " },\n", + " Conv_1: {\n", + " kernel: DeviceArray([[[[ 3.60345989e-02, -1.80624407e-02, -9.17060301e-03, ...,\n", + " -1.57420221e-03, -3.21763046e-02, 3.03349812e-02],\n", + " [-4.88650911e-02, 2.21489817e-02, 1.55314393e-02, ...,\n", + " 1.22555168e-02, 2.84199193e-02, 2.09916160e-02],\n", + " [ 5.93110994e-02, -5.48763536e-02, -3.18756956e-03, ...,\n", + " 3.08737922e-02, -8.48920736e-03, -2.10110340e-02],\n", + " ...,\n", + " [ 4.99920659e-02, -6.42840192e-02, -2.96061370e-03, ...,\n", + " 1.10166790e-02, 1.05426693e-02, 2.54060999e-02],\n", + " [ 5.33394516e-02, 4.65421192e-02, 1.49851209e-02, ...,\n", + " 5.68490475e-03, -1.30498027e-02, -1.01780500e-02],\n", + " [ 6.59535229e-02, 9.09722038e-03, -2.24497523e-02, ...,\n", + " 7.19328318e-03, 5.78204077e-03, 5.39870039e-02]],\n", + " \n", + " [[-3.70773338e-02, -3.87179144e-02, 1.63091160e-02, ...,\n", + " 2.97876936e-03, 1.07463440e-02, -1.82296222e-04],\n", + " [-2.36669872e-02, -5.58980182e-03, -2.70987861e-02, ...,\n", + " 3.27060483e-02, 5.49956113e-02, -3.22447494e-02],\n", + " [ 3.49065661e-02, 2.02603359e-02, -4.39962931e-03, ...,\n", + " 1.14598181e-02, -1.17074130e-02, 4.88227000e-03],\n", + " ...,\n", + " [-3.37145887e-02, 4.31741215e-02, 4.60439175e-03, ...,\n", + " -1.50344986e-02, -2.81697568e-02, -2.23013163e-02],\n", + " [ 4.66451123e-02, 2.47632357e-04, 1.16665391e-02, ...,\n", + " -4.74373903e-03, 7.42866658e-03, -5.94518706e-02],\n", + " [ 1.07875392e-02, -1.24657005e-02, -3.20048593e-02, ...,\n", + " 1.44016631e-02, -4.46076095e-02, -4.64649275e-02]],\n", + " \n", + " [[-1.87598765e-02, -1.26873795e-02, 2.22337674e-02, ...,\n", + " -1.86812971e-02, 3.24476734e-02, -2.59078629e-02],\n", + " [ 2.18403228e-02, -2.43654586e-02, -5.68524711e-02, ...,\n", + " -3.14403288e-02, 1.45379817e-02, -3.71383354e-02],\n", + " [ 5.04917093e-02, 3.70514989e-02, -2.47972971e-03, ...,\n", + " 9.21094697e-03, 6.62889564e-03, 5.43021150e-02],\n", + " ...,\n", + " [ 5.53301275e-02, 8.09799600e-03, -7.37548014e-03, ...,\n", + " -3.22527029e-02, 9.47414152e-03, 1.66119877e-02],\n", + " [ 3.12854238e-02, -9.37583914e-04, -2.83447886e-03, ...,\n", + " 6.54492676e-02, -1.27467439e-02, -2.87838411e-02],\n", + " [ 4.77980748e-02, -1.19424276e-02, 1.74422693e-02, ...,\n", + " -3.70779335e-02, 2.49283463e-02, 5.49281761e-03]]],\n", + " \n", + " \n", + " [[[ 2.17519812e-02, -3.88661511e-02, 3.92616428e-02, ...,\n", + " 5.27783893e-02, 9.30878986e-03, -1.11992378e-02],\n", + " [-4.48921323e-02, 2.66084112e-02, -3.08889486e-02, ...,\n", + " -2.99759768e-03, 2.66164411e-02, -2.56100800e-02],\n", + " [ 1.48384757e-02, 2.49346737e-02, 2.10087299e-02, ...,\n", + " 1.19575774e-02, -4.37101051e-02, 1.59911637e-03],\n", + " ...,\n", + " [ 4.79628369e-02, 2.65846997e-02, -5.52016264e-03, ...,\n", + " 8.03794805e-03, 1.57203991e-02, -3.72503996e-02],\n", + " [ 3.36065628e-02, 5.82088120e-02, -4.79097627e-02, ...,\n", + " 1.47540374e-02, -4.82356735e-02, 2.53046560e-03],\n", + " [-1.85025409e-02, 1.19747752e-02, -2.92210910e-03, ...,\n", + " -3.48893404e-02, -4.21325751e-02, 1.10742254e-02]],\n", + " \n", + " [[-2.33020280e-02, -5.33820689e-02, 6.04530687e-05, ...,\n", + " -1.82941053e-02, 2.49706823e-02, 3.89664173e-02],\n", + " [ 3.02372547e-03, 3.97237539e-02, -4.37463261e-02, ...,\n", + " -4.79917275e-03, -3.19175012e-02, -5.08953705e-02],\n", + " [-3.39236967e-02, -6.98743481e-03, 2.17721332e-02, ...,\n", + " -4.89775054e-02, -5.73655218e-02, 3.42595093e-02],\n", + " ...,\n", + " [ 3.00649554e-02, -8.40300415e-03, -7.04725645e-03, ...,\n", + " -3.24916989e-02, 1.48367360e-02, 3.87442624e-03],\n", + " [ 1.28357522e-02, 1.16875945e-02, 2.62607671e-02, ...,\n", + " 2.08278652e-02, -2.04269513e-02, 2.65005231e-02],\n", + " [ 1.62072293e-02, 1.26523693e-04, 6.51293248e-02, ...,\n", + " -3.86682414e-02, -1.51754823e-03, -1.52847609e-02]],\n", + " \n", + " [[-1.05653480e-02, -3.87853314e-03, 3.05805751e-03, ...,\n", + " 3.76726035e-03, 4.94486131e-02, -5.60158156e-02],\n", + " [ 4.87619005e-02, -4.03809361e-03, 5.23778163e-02, ...,\n", + " -2.93569081e-02, 3.27635743e-02, 2.99357139e-02],\n", + " [-2.07389537e-02, -3.72745059e-02, 1.10834790e-02, ...,\n", + " -6.21929392e-02, 2.98373811e-02, 1.99949909e-02],\n", + " ...,\n", + " [ 3.27900052e-02, 5.98487668e-02, 4.71824668e-02, ...,\n", + " 8.78540706e-03, -6.14813250e-03, -3.14074531e-02],\n", + " [-9.78266541e-03, -5.90384342e-02, -5.13021387e-02, ...,\n", + " -1.38909575e-02, -2.51286477e-02, 5.18727489e-03],\n", + " [-3.64494771e-02, -4.27394621e-02, -3.23385722e-03, ...,\n", + " -3.18818376e-03, -5.87067306e-02, -6.36329651e-02]]],\n", + " \n", + " \n", + " [[[-8.47443007e-03, -4.90162708e-02, -1.27213998e-02, ...,\n", + " -7.46757211e-03, -2.31349524e-02, -3.39074843e-02],\n", + " [ 3.42674591e-02, -4.26242724e-02, 8.94031115e-03, ...,\n", + " 5.81473745e-02, 3.11985286e-03, -7.18289101e-03],\n", + " [-4.96962480e-02, -1.75054520e-02, 1.33445645e-02, ...,\n", + " 1.52927013e-02, 7.62744015e-03, 1.03257475e-02],\n", + " ...,\n", + " [-2.10905168e-02, 8.50216951e-04, -2.00528335e-02, ...,\n", + " 6.31971285e-03, 2.48127226e-02, -1.87783968e-02],\n", + " [-2.23999172e-02, 1.26594771e-02, 6.04727156e-02, ...,\n", + " -6.21030182e-02, 5.47166280e-02, -1.63479019e-02],\n", + " [ 9.98661667e-03, -2.33470928e-03, -2.09867079e-02, ...,\n", + " -3.46814878e-02, 2.51958854e-02, -1.26770735e-02]],\n", + " \n", + " [[-1.97642110e-02, 4.73201126e-02, 4.43987735e-02, ...,\n", + " 2.50004977e-02, -2.66237138e-03, -2.26721596e-02],\n", + " [ 6.78458950e-03, 1.61966719e-02, 2.98016015e-02, ...,\n", + " -5.48582349e-04, -4.17552330e-02, -4.90760766e-02],\n", + " [-3.85830216e-02, 6.56447783e-02, 6.06308272e-03, ...,\n", + " 7.21618021e-03, 3.87805365e-02, -4.35139164e-02],\n", + " ...,\n", + " [ 1.90136768e-02, 5.99899096e-03, 2.55533494e-02, ...,\n", + " -4.07748334e-02, -2.88647264e-02, 3.18611637e-02],\n", + " [ 2.06444710e-02, 3.52281965e-02, -3.74659821e-02, ...,\n", + " -2.47550700e-02, 7.98308384e-03, -1.17612109e-02],\n", + " [ 1.07358827e-03, -1.51244262e-02, 4.03979719e-02, ...,\n", + " 1.38022611e-02, -1.51816355e-02, 1.29964007e-02]],\n", + " \n", + " [[ 4.76968586e-02, -1.51051348e-02, 1.60713233e-02, ...,\n", + " -1.36397481e-02, -4.37372923e-02, -6.09542839e-02],\n", + " [-7.29514472e-03, 4.64752316e-03, 7.45077617e-03, ...,\n", + " -5.41699976e-02, 1.31481951e-02, -3.93584818e-02],\n", + " [ 5.68703189e-02, -1.18301352e-02, 2.59039719e-02, ...,\n", + " 4.63015288e-02, -1.92511063e-02, -2.26892084e-02],\n", + " ...,\n", + " [ 3.30666825e-02, -1.54788038e-02, 1.28783807e-02, ...,\n", + " -4.02262770e-02, -4.79671173e-03, 3.56641621e-03],\n", + " [-1.72251593e-02, -1.34579502e-02, -4.67023253e-02, ...,\n", + " 2.73214784e-02, 3.60498987e-02, -4.09855619e-02],\n", + " [ 2.44301874e-02, -3.01268771e-02, 4.78487536e-02, ...,\n", + " -3.52564305e-02, -4.78573143e-02, -1.05750179e-02]]]], dtype=float32),\n", + " bias: DeviceArray([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32),\n", + " },\n", + " Dense_0: {\n", + " kernel: DeviceArray([[ 0.00046263, -0.01366332, -0.01965372, ..., -0.01099672,\n", + " 0.00353724, 0.02137254],\n", + " [ 0.00497269, -0.00472533, -0.01134661, ..., -0.00669816,\n", + " -0.00842239, -0.02093654],\n", + " [ 0.03009967, 0.00874493, 0.00541858, ..., 0.02110084,\n", + " -0.00377029, -0.02680648],\n", + " ...,\n", + " [-0.00427975, -0.00622658, -0.00404923, ..., 0.00897263,\n", + " 0.00962972, -0.00889211],\n", + " [-0.01013104, -0.00755738, 0.01790517, ..., -0.0158775 ,\n", + " -0.01318363, -0.01386497],\n", + " [ 0.02921496, -0.01328129, 0.01899061, ..., 0.03121358,\n", + " 0.02289535, 0.01520465]], dtype=float32),\n", + " bias: DeviceArray([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", + " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32),\n", + " },\n", + " Dense_1: {\n", + " kernel: DeviceArray([[ 0.09170318, 0.03614756, 0.02535907, ..., 0.0762569 ,\n", + " -0.0044047 , 0.0330187 ],\n", + " [-0.07894796, -0.04608489, 0.00269128, ..., 0.02883765,\n", + " 0.0087204 , 0.05653023],\n", + " [ 0.03623685, 0.00995611, -0.04651761, ..., -0.05153248,\n", + " 0.02583626, -0.01592986],\n", + " ...,\n", + " [ 0.02275847, 0.08729529, -0.0204505 , ..., 0.09371196,\n", + " 0.06301026, 0.07643232],\n", + " [-0.02012173, 0.06339496, 0.01162816, ..., 0.07186693,\n", + " -0.05107251, -0.01174682],\n", + " [ 0.0267258 , 0.12019555, -0.06517248, ..., 0.00200361,\n", + " 0.02145562, -0.10603211]], dtype=float32),\n", + " bias: DeviceArray([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32),\n", + " },\n", + "})" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data[\"params\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "from neural_tangents import stax" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [], + "source": [ + "model = stax.serial(\n", + " stax.Dense(12),\n", + " stax.Relu(),\n", + " stax.Dense(12),\n", + " stax.Relu(),\n", + " stax.Dense(1)\n", + ")\n", + "small_model = stax.serial(\n", + " stax.Dense(12),\n", + " stax.Relu(),\n", + " stax.Dense(12),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "metadata": {}, + "outputs": [], + "source": [ + "_, params = model[0](key, (9,))" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [], + "source": [ + "tst = np.random.uniform(size=(9,))" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DeviceArray([-0.00011978], dtype=float32)" + ] + }, + "execution_count": 63, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model[1](params, tst)" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[(DeviceArray([[ 0.36902574, -0.60083115, 0.44724423, -1.6919093 ,\n", + " -1.4313933 , -0.9526142 , 1.5278178 , 1.2445855 ,\n", + " -0.43463358, 0.1921436 , -0.51044935, 1.3951449 ],\n", + " [ 2.6688576 , 0.27140683, 0.7197667 , 1.7743273 ,\n", + " -0.12421109, 2.8912168 , 0.8812158 , -0.59462166,\n", + " 0.75510025, -0.82257533, -0.69549227, 0.11659083],\n", + " [ 0.7083261 , 0.0195996 , -0.96690786, -1.9975785 ,\n", + " -1.2147071 , 0.49638763, -0.882764 , 0.06965447,\n", + " 0.8740023 , -1.7946595 , -1.9639919 , 0.11853004],\n", + " [-0.04729311, -0.6084024 , -1.151995 , 1.8441046 ,\n", + " -1.0241059 , 0.29526377, -1.6498058 , 0.35885495,\n", + " -1.8048742 , -0.45532495, 0.86956114, 0.7042805 ],\n", + " [ 0.7024425 , -0.46502215, -1.8246489 , 0.03024639,\n", + " -0.09241743, 0.6734406 , -1.4146967 , 0.83981234,\n", + " 1.6875414 , 1.6478448 , -0.14617752, 0.73602235],\n", + " [ 0.11008073, 0.59370816, -0.47999948, -0.91698813,\n", + " 1.4644262 , -1.0099113 , 0.42220187, 0.57421154,\n", + " -1.6171031 , -0.4017566 , 1.6489285 , 0.18311332],\n", + " [ 0.9582781 , -1.557588 , -0.19959736, -0.15300396,\n", + " 1.4889674 , -0.253901 , 0.37354448, 1.5264944 ,\n", + " -2.0760427 , -0.5360183 , 0.61596334, 0.24212281],\n", + " [ 1.1130695 , -0.9824064 , -2.977521 , 0.74219394,\n", + " -1.1848937 , -0.12566446, -0.52775025, 0.28832507,\n", + " 1.53694 , 0.731218 , 1.3954059 , 1.9324546 ],\n", + " [-0.2378023 , 0.3914802 , -0.62872165, 0.3234975 ,\n", + " -0.61664855, -0.37052616, 0.4133723 , 0.3731773 ,\n", + " -1.6918828 , -0.40905452, 1.9508578 , -0.66678524]], dtype=float32),\n", + " None),\n", + " (),\n", + " (DeviceArray([[ 0.31568572, -0.78239375, 1.8139715 , 0.7123382 ,\n", + " 0.42159593, 1.4614258 , -1.2938766 , -0.2708335 ,\n", + " -0.8100683 , 0.42933106, 0.34290957, 2.5893972 ],\n", + " [-1.7116116 , -1.1352941 , -0.5111037 , -1.4054971 ,\n", + " -0.32070762, -1.42568 , 0.11512792, -0.1347293 ,\n", + " -0.71804935, 0.05384207, -0.78869826, -0.5870854 ],\n", + " [-0.32054782, 1.2623926 , -0.27968442, -0.591598 ,\n", + " -0.4438807 , -0.8524836 , 1.3454165 , 1.0766404 ,\n", + " 0.45279294, 0.83987784, 0.03888967, -0.19251114],\n", + " [-0.77271 , -0.04992997, -0.3330754 , 0.3508638 ,\n", + " -0.10676403, -0.58075947, 0.02378667, -1.9671077 ,\n", + " 0.0314757 , -0.82272756, -0.00710218, -1.1613281 ],\n", + " [-1.5903914 , 1.1984488 , -0.27399755, -0.06564435,\n", + " -0.62434304, 0.18636021, -0.29711506, 1.8457091 ,\n", + " -1.0972382 , 0.4340854 , 0.5363783 , -0.8186496 ],\n", + " [-0.19632275, 0.9917772 , 0.48805034, -0.83440447,\n", + " 0.18030544, 0.5788825 , 1.2664341 , 0.9014271 ,\n", + " 0.20820503, -0.60801953, -0.1659515 , -1.5893393 ],\n", + " [-1.0448416 , -1.0183587 , 1.1974787 , -1.5628817 ,\n", + " -1.1902788 , 0.02965477, -0.11841193, 0.9383633 ,\n", + " -0.6725073 , 1.2212063 , 0.8176451 , -0.25271893],\n", + " [-1.2024895 , -0.1259256 , -1.1257274 , -0.0786797 ,\n", + " -1.5528514 , -0.13004757, 0.49707723, -0.7027365 ,\n", + " 0.48414403, -2.0060594 , 0.5135008 , -0.800257 ],\n", + " [-0.21854441, -1.3389152 , -1.1716666 , 1.6518806 ,\n", + " 0.5396923 , 1.181409 , -0.1754383 , 1.391895 ,\n", + " -0.12838942, -0.28058812, -0.6165152 , -1.2462329 ],\n", + " [-0.9806118 , -0.0681522 , -0.43525243, 0.08544233,\n", + " 1.0447279 , 0.49811652, -1.262155 , -0.6817887 ,\n", + " -0.80958515, 0.31914094, -1.1228462 , -0.87152374],\n", + " [ 0.51965606, 0.7304284 , 0.18613489, 0.38158152,\n", + " -0.93155473, 0.37154427, 0.32466838, 0.65689003,\n", + " 0.7206714 , -0.4563752 , -0.5785838 , 1.020707 ],\n", + " [-0.28219694, -0.88026786, 0.4212645 , -1.3743248 ,\n", + " 1.375825 , 1.9244089 , -0.40021232, 0.4462603 ,\n", + " 1.5846783 , -0.4113239 , -0.7848289 , 0.5002319 ]], dtype=float32),\n", + " None)]" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "params[:-2]" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [], + "source": [ + "a = small_model[1](params, tst)" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [], + "source": [ + "b = small_model[1](params, tst)" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DeviceArray([ True, True, True, True, True, True, True, True,\n", + " True, True, True, True], dtype=bool)" + ] + }, + "execution_count": 80, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a == b" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(.init_fun(rng, input_shape)>,\n", + " .apply_fun(params, inputs, **kwargs)>,\n", + " .kernel_fn_any(x1_or_kernel: Union[List[jax._src.numpy.ndarray.ndarray], Tuple[jax._src.numpy.ndarray.ndarray, ...], jax._src.numpy.ndarray.ndarray, List[neural_tangents._src.utils.kernel.Kernel], Tuple[neural_tangents._src.utils.kernel.Kernel, ...], neural_tangents._src.utils.kernel.Kernel], x2: Union[List[jax._src.numpy.ndarray.ndarray], Tuple[jax._src.numpy.ndarray.ndarray, ...], jax._src.numpy.ndarray.ndarray, NoneType] = None, get: Union[Tuple[str, ...], str, NoneType] = None, *, pattern: Union[Tuple[Union[jax._src.numpy.ndarray.ndarray, NoneType], Union[jax._src.numpy.ndarray.ndarray, NoneType]], NoneType] = None, mask_constant: Union[float, NoneType] = None, diagonal_batch: Union[bool, NoneType] = None, diagonal_spatial: Union[bool, NoneType] = None, **kwargs)>)" + ] + }, + "execution_count": 84, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -396,7 +933,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.8.12" } }, "nbformat": 4, diff --git a/examples/scripts/SO2_extraction.py b/examples/scripts/SO2_extraction.py index c75cf25..3bf488a 100644 --- a/examples/scripts/SO2_extraction.py +++ b/examples/scripts/SO2_extraction.py @@ -2,10 +2,11 @@ Python module to show generator extraction of SO(2) Lie algebra generators. """ -from symsuite.test_systems.so2_data import SO2 -from symsuite.generators.generators import GeneratorExtraction import numpy as np +from symsuite.generators.generators import GeneratorExtraction +from symsuite.test_systems.so2_data import SO2 + def generator_extraction(): """ diff --git a/examples/scripts/SO3_extraction.py b/examples/scripts/SO3_extraction.py index c7cbf90..6743e69 100644 --- a/examples/scripts/SO3_extraction.py +++ b/examples/scripts/SO3_extraction.py @@ -2,8 +2,8 @@ Python module to show generator extraction of SO(3) Lie algebra generators. """ -from symsuite.test_systems.so3_data import SO3 from symsuite.generators.generators import GeneratorExtraction +from symsuite.test_systems.so3_data import SO3 def generator_extraction(): diff --git a/examples/scripts/double_well_investigation.py b/examples/scripts/double_well_investigation.py index 937a9ac..5414207 100644 --- a/examples/scripts/double_well_investigation.py +++ b/examples/scripts/double_well_investigation.py @@ -8,10 +8,10 @@ network and visualizing its embedding layer using TSNE. """ -from symsuite.test_systems.double_well_potential import DoubleWellPotential from symsuite.models.dense_model import DenseModel from symsuite.symmetry_groups.data_clustering import DataCluster from symsuite.symmetry_groups.group_detection import GroupDetection +from symsuite.test_systems.double_well_potential import DoubleWellPotential def main(): diff --git a/requirements.txt b/requirements.txt index 93ef214..a9c21b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ IPython pandoc numpydoc pre-commit +jax diff --git a/setup.py b/setup.py index a1c9cc8..f16041d 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,10 @@ Setup.py file for the SymDet package. """ -import setuptools from os import path +import setuptools + here = path.abspath(path.dirname(__file__)) with open(path.join(here, "requirements.txt")) as requirements_file: # Parse requirements.txt, ignoring any commented-out lines. diff --git a/symsuite/__init__.py b/symsuite/__init__.py index a7d749e..8465e94 100644 --- a/symsuite/__init__.py +++ b/symsuite/__init__.py @@ -2,12 +2,13 @@ __init__ file for the symsuite package """ import os + from symsuite.data.double_well_potential import DoubleWellPotential -from symsuite.ml_models.dense_model import DenseModel -from symsuite.symmetry_group_extraction.group_detection import GroupDetection -from symsuite.generator_extraction.generators import GeneratorExtraction from symsuite.data.so2_data import SO2 from symsuite.data.so3_data import SO3 +from symsuite.generator_extraction.generators import GeneratorExtraction +from symsuite.ml_models.dense_model import DenseModel +from symsuite.symmetry_group_extraction.group_detection import GroupDetection -__all__ = ['DoubleWellPotential', 'DenseModel', 'GroupDetection', 'SO2', 'SO3'] +__all__ = ["DoubleWellPotential", "DenseModel", "GroupDetection", "SO2", "SO3"] os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" diff --git a/symsuite/accuracy_functions/__init__.py b/symsuite/accuracy_functions/__init__.py new file mode 100644 index 0000000..b15ca28 --- /dev/null +++ b/symsuite/accuracy_functions/__init__.py @@ -0,0 +1,30 @@ +""" +SymSuite +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +init function for the accuracy functions. +""" +from symsuite.accuracy_functions.accuracy_function import AccuracyFunction +from symsuite.accuracy_functions.label_accuracy import LabelAccuracy + +__all__ = [AccuracyFunction.__name__, LabelAccuracy.__name__] diff --git a/symsuite/accuracy_functions/accuracy_function.py b/symsuite/accuracy_functions/accuracy_function.py new file mode 100644 index 0000000..d3a3d24 --- /dev/null +++ b/symsuite/accuracy_functions/accuracy_function.py @@ -0,0 +1,53 @@ +""" +ZnRND: A zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Parent class for the accuracy functions. +""" +import jax.numpy as np + + +class AccuracyFunction: + """ + Class for computing accuracy. + """ + + def __call__(self, predictions: np.array, targets: np.array) -> float: + """ + Accuracy function call method. + + Parameters + ---------- + predictions : np.array + First set of points to be compared. + targets : np.array + Second points to compare. This will be passed through any + pre-processing of the child classes. + + Returns + ------- + accuracy : float + Accuracy of the points. + """ + raise NotImplementedError("Implemented in child class.") diff --git a/symsuite/accuracy_functions/label_accuracy.py b/symsuite/accuracy_functions/label_accuracy.py new file mode 100644 index 0000000..f8d402e --- /dev/null +++ b/symsuite/accuracy_functions/label_accuracy.py @@ -0,0 +1,65 @@ +""" +ZnRND: A zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Compute the one hot accuracy between two points. +""" +import jax.numpy as np + +from symsuite.accuracy_functions.accuracy_function import AccuracyFunction + + +class LabelAccuracy(AccuracyFunction): + """ + Compute the one hot accuracy between two points. + """ + + def __init__(self, num_class: int): + """ + Constructor for the one hot accuracy. + + Parameters + ---------- + num_class : int + Number of classes in the one hot encoding. + """ + self.num_classes = num_class + + def __call__(self, predictions: np.array, targets: np.array) -> float: + """ + Accuracy function call method. + + Parameters + ---------- + predictions : np.array + First set of points to be compared. + targets : np.array + Second points to compare. Does not require one hot encoding. + + Returns + ------- + accuracy : float + Accuracy of the points. + """ + return np.mean(np.argmax(predictions, -1) == targets) diff --git a/symsuite/analysis/model_visualization.py b/symsuite/analysis/model_visualization.py index 5dd9bde..91b58b3 100644 --- a/symsuite/analysis/model_visualization.py +++ b/symsuite/analysis/model_visualization.py @@ -11,8 +11,8 @@ Visualize the NN models in different ways """ import matplotlib.pyplot as plt -from sklearn.manifold import TSNE import numpy as np +from sklearn.manifold import TSNE class Visualizer: @@ -42,11 +42,9 @@ def __init__(self, data, colour_map): self.data = data self.colour_map = colour_map - def tsne_visualization(self, - perplexity=50, - n_components=2, - plot: bool = True, - save: bool = False) -> np.ndarray: + def tsne_visualization( + self, perplexity=50, n_components=2, plot: bool = True, save: bool = False + ) -> np.ndarray: """ Display a TSNE representation of the models embedding layer @@ -70,9 +68,9 @@ def tsne_visualization(self, See the theory documentation for a full overview of these parameters, particularly in the case of the TSNE values. """ - tsne_model = TSNE(n_components=n_components, - perplexity=perplexity, - random_state=1) + tsne_model = TSNE( + n_components=n_components, perplexity=perplexity, random_state=1 + ) tsne_representation = tsne_model.fit_transform(self.data) if plot: @@ -80,7 +78,7 @@ def tsne_visualization(self, tsne_representation[:, 0], tsne_representation[:, 1], c=self.colour_map, - marker='.', + marker=".", cmap="viridis", vmax=11, vmin=-1, diff --git a/symsuite/data/__init__.py b/symsuite/data/__init__.py index 33e5766..78a8631 100644 --- a/symsuite/data/__init__.py +++ b/symsuite/data/__init__.py @@ -7,4 +7,4 @@ Copyright Contributors to the Zincware Project. Description: __init__ file for the data package. -""" \ No newline at end of file +""" diff --git a/symsuite/data/data_generator.py b/symsuite/data/data_generator.py index 28b4185..a6055ae 100644 --- a/symsuite/data/data_generator.py +++ b/symsuite/data/data_generator.py @@ -11,8 +11,9 @@ """ import abc from typing import Union + +import jax.numpy as jnp import numpy as np -import tensorflow as tf class DataGenerator(metaclass=abc.ABCMeta): diff --git a/symsuite/data/double_well_potential.py b/symsuite/data/double_well_potential.py index 6599e7b..7d56350 100644 --- a/symsuite/data/double_well_potential.py +++ b/symsuite/data/double_well_potential.py @@ -8,14 +8,17 @@ Description: Example data generator for the double well potential. """ -from symsuite.data.data_generator import DataGenerator -from symsuite.utils.data_clustering import range_binning from typing import Union -import numpy as np -import tensorflow as tf + +import jax +import jax.numpy as jnp import matplotlib.pyplot as plt +import numpy as np from tqdm import tqdm +from symsuite.data.data_generator import DataGenerator +from symsuite.utils.data_clustering import range_binning + class DoubleWellPotential(DataGenerator): """ @@ -43,8 +46,8 @@ class DoubleWellPotential(DataGenerator): .. math:: V = -a \cdot (x^{2} + y^{2}) + (x^{2} + y^{2})^{2} - We will require the data to be stored as x,y coordinates in order to facilitate the generator extraction in this - part of the process. + We will require the data to be stored as x,y coordinates in order to facilitate + the generator extraction in this part of the process. """ def __init__(self, a: float = 2.3): @@ -66,8 +69,8 @@ def _double_well(self): ------- """ - square_radii = tf.reduce_sum(tf.math.square(self.domain), 1) - self.image = -self.a * square_radii + tf.square(square_radii) + square_radii = jnp.sum(self.domain ** 2, axis=1) + self.image = -self.a * square_radii + square_radii ** 2 def _pick_points(self, n_points: int, min_val: float = 0, max_val: float = 1.6): """ @@ -84,11 +87,13 @@ def _pick_points(self, n_points: int, min_val: float = 0, max_val: float = 1.6): Returns ------- - + Updates the class attributes. """ - self.domain = tf.random.uniform(shape=(n_points, 2), - minval=min_val, - maxval=max_val) + key = jax.random.PRNGKey(0) + key, subkey = jax.random.split(key) + self.domain = jax.random.uniform( + subkey, shape=(n_points, 2), minval=min_val, maxval=max_val + ) def load_data(self, points: Union[int, np.ndarray], save: bool = False): """ @@ -97,9 +102,10 @@ def load_data(self, points: Union[int, np.ndarray], save: bool = False): Parameters ---------- points : Union[int, np.ndarray] - Points to generate, either an np.ndarray or an integer. If an integer, N points will be generated, if - an array, it will either be treated as input to a function to generate values or those indices will be - drawn from a pool. + Points to generate, either an np.ndarray or an integer. If an integer, + N points will be generated, if an array, it will either be treated as + input to a function to generate values or those indices will be drawn + from a pool. save : bool If true, save the data after generating it. @@ -114,7 +120,7 @@ def load_data(self, points: Union[int, np.ndarray], save: bool = False): # set domain and generate image data. else: - self.domain = tf.convert_to_tensor(points) + self.domain = jnp.array(points) self._double_well() def plot_clusters(self, save: bool = False): @@ -131,15 +137,17 @@ def plot_clusters(self, save: bool = False): """ self.plot_data(show=False) - for i, item in tqdm(enumerate(self.clustered_data), ncols=70, total=len(self.clustered_data)): - r = tf.norm(self.clustered_data[item]['domain'], axis=1) - v = self.clustered_data[item]['image'] - plt.plot(r, v, '.', label=f"Class {i}", markersize=15) + for i, item in tqdm( + enumerate(self.clustered_data), ncols=70, total=len(self.clustered_data) + ): + r = jnp.linalg.norm(self.clustered_data[item]["domain"], axis=1) + v = self.clustered_data[item]["image"] + plt.plot(r, v, ".", label=f"Class {i}", markersize=15) - plt.legend(loc='center left', bbox_to_anchor=(1, 0.5)) + plt.legend(loc="center left", bbox_to_anchor=(1, 0.5)) if save: - plt.savefig('Clusters.png', dpi=600) + plt.savefig("Clusters.png", dpi=600) plt.show() @@ -160,18 +168,20 @@ def plot_data(self, save: bool = False, show: bool = True): if self.domain is None: self._pick_points(1000, min_val=0, max_val=1.2) self._double_well() - plt.plot(tf.norm(self.domain, axis=1), self.image, '.') - plt.xlabel('r') - plt.ylabel('V') + plt.plot(jnp.linalg.norm(self.domain, axis=1), self.image, ".") + plt.xlabel("r") + plt.ylabel("V") plt.xlim(-0.03, 1.7) plt.ylim(-1.5, 1.3) plt.grid() if save: - plt.savefig(f'Double_Well_{len(self.domain)}.svg', dpi=600, format='dpi') + plt.savefig(f"Double_Well_{len(self.domain)}.svg", dpi=600, format="dpi") if show: plt.show() - def build_clusters(self, value_range: list = None, bin_operation: list = None, representatives=1000): + def build_clusters( + self, value_range: list = None, bin_operation: list = None, representatives=1000 + ): """ Split the raw function data into classes. @@ -191,7 +201,8 @@ def build_clusters(self, value_range: list = None, bin_operation: list = None, r Notes ----- - In the double well potential we can simply use the range_binning clustering algorithm. + In the double well potential we can simply use the range_binning clustering + algorithm. """ # Replace None type parameters. if bin_operation is None: @@ -207,8 +218,10 @@ def build_clusters(self, value_range: list = None, bin_operation: list = None, r print("Loading additional data.") self.load_data(n_classes * representatives * 1000) - self.clustered_data = range_binning(image=self.image, - domain=self.domain, - value_range=value_range, - bin_operation=bin_operation, - representatives=representatives) + self.clustered_data = range_binning( + image=self.image, + domain=self.domain, + value_range=value_range, + bin_operation=bin_operation, + representatives=representatives, + ) diff --git a/symsuite/data/so2_data.py b/symsuite/data/so2_data.py index a5afd09..37d673e 100644 --- a/symsuite/data/so2_data.py +++ b/symsuite/data/so2_data.py @@ -8,10 +8,12 @@ Description: Module for the computation of so2 data """ -from symsuite.data.data_generator import DataGenerator from typing import Union -import numpy as np + import matplotlib.pyplot as plt +import numpy as np + +from symsuite.data.data_generator import DataGenerator class SO2(DataGenerator): @@ -74,9 +76,9 @@ def _circle(self, points: int): """ if self.noise: - self.radial_values = np.random.uniform(self.radius - self.variance, - self.radius + self.variance, - points) + self.radial_values = np.random.uniform( + self.radius - self.variance, self.radius + self.variance, points + ) else: self.radial_values = self.radius @@ -110,7 +112,9 @@ def load_data(self, points: Union[int, np.ndarray], save: bool = False): # set domain and generate image data. else: - raise ValueError(f"Type {type(points)} is not valid for this data generator, try an integer") + raise ValueError( + f"Type {type(points)} is not valid for this data generator, try an integer" + ) def plot_data(self, save: bool = False, show: bool = True): """ diff --git a/symsuite/data/so3_data.py b/symsuite/data/so3_data.py index 916e9a3..6670f23 100644 --- a/symsuite/data/so3_data.py +++ b/symsuite/data/so3_data.py @@ -8,10 +8,12 @@ Description: Module for the computation of so3 data """ -from symsuite.data.data_generator import DataGenerator from typing import Union -import numpy as np + import matplotlib.pyplot as plt +import numpy as np + +from symsuite.data.data_generator import DataGenerator class SO3(DataGenerator): @@ -74,9 +76,9 @@ def _sphere(self, points: int): """ if self.noise: - self.radial_values = np.random.uniform(self.radius - self.variance, - self.radius + self.variance, - points) + self.radial_values = np.random.uniform( + self.radius - self.variance, self.radius + self.variance, points + ) else: self.radial_values = self.radius @@ -112,7 +114,9 @@ def load_data(self, points: Union[int, np.ndarray], save: bool = False): # set domain and generate image data. else: - raise ValueError(f"Type {type(points)} is not valid for this data generator, try an integer") + raise ValueError( + f"Type {type(points)} is not valid for this data generator, try an integer" + ) def plot_data(self, save: bool = False, show: bool = True): """ @@ -133,15 +137,15 @@ def plot_data(self, save: bool = False, show: bool = True): fig = plt.figure() ax = fig.add_subplot(111, projection="3d") - ax.scatter(self.domain[:, 0], - self.domain[:, 1], - self.domain[:, 2], - marker=".", - color="k") + ax.scatter( + self.domain[:, 0], + self.domain[:, 1], + self.domain[:, 2], + marker=".", + color="k", + ) if save: - plt.savefig(f"SO(2)_{len(self.domain)}.svg", - dpi=800, - format="svg") + plt.savefig(f"SO(2)_{len(self.domain)}.svg", dpi=800, format="svg") plt.show() def build_clusters(self, **kwargs): diff --git a/symsuite/distance_metrics/__init__.py b/symsuite/distance_metrics/__init__.py new file mode 100644 index 0000000..7ac5f7a --- /dev/null +++ b/symsuite/distance_metrics/__init__.py @@ -0,0 +1,39 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +distance metric module +""" +from symsuite.distance_metrics.angular_distance import AngularDistance +from symsuite.distance_metrics.cosine_distance import CosineDistance +from symsuite.distance_metrics.distance_metric import DistanceMetric +from symsuite.distance_metrics.hyper_sphere_distance import HyperSphere +from symsuite.distance_metrics.l_p_norm import LPNorm +from symsuite.distance_metrics.mahalanobis_distance import MahalanobisDistance +from symsuite.distance_metrics.order_n_difference import OrderNDifference + +__all__ = [ + DistanceMetric.__name__, + CosineDistance.__name__, + AngularDistance.__name__, + LPNorm.__name__, + OrderNDifference.__name__, + MahalanobisDistance.__name__, + HyperSphere.__name__, +] diff --git a/symsuite/distance_metrics/angular_distance.py b/symsuite/distance_metrics/angular_distance.py new file mode 100644 index 0000000..3817c16 --- /dev/null +++ b/symsuite/distance_metrics/angular_distance.py @@ -0,0 +1,78 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Compute the angular distance between two points normalized by the point density in the +circle. +""" +import jax.numpy as np + +from symsuite.distance_metrics.distance_metric import DistanceMetric + + +class AngularDistance(DistanceMetric): + """ + Class for the angular distance metric. + """ + + def __init__(self, points: int = None): + """ + Constructor for the angular distance metric. + + Parameters + ---------- + points : int + Number of points in the circle. If None, normalization by pi is used. + """ + if points is None: + self.normalization = np.pi + elif type(points) is int and points > 0: + self.normalization = points / np.pi + else: + raise ValueError("Invalid points input.") + + def __call__(self, point_1: np.ndarray, point_2: np.ndarray, **kwargs): + """ + Call the distance metric. + + Distance between points in the point_1 tensor will be computed between those in + the point_2 tensor element-wise. Therefore, we will have: + + point_1[i] - point_2[i] for all i. + + Parameters + ---------- + point_1 : np.ndarray (n_points, point_dimension) + First set of points in the comparison. + point_2 : np.ndarray (n_points, point_dimension) + Second set of points in the comparison. + kwargs + Miscellaneous keyword arguments for the specific metric. + + Returns + ------- + d(point_1, point_2) : tf.tensor, shape=(n_points, 1) + Array of distances for each point. + """ + numerator = np.einsum("ij, ij -> i", point_1, point_2) + denominator = np.sqrt( + np.einsum("ij, ij -> i", point_1, point_1) + * np.einsum("ij, ij -> i", point_2, point_2) + ) + return np.arccos(abs(np.divide(numerator, denominator))) / self.normalization diff --git a/symsuite/distance_metrics/cosine_distance.py b/symsuite/distance_metrics/cosine_distance.py new file mode 100644 index 0000000..89d0f57 --- /dev/null +++ b/symsuite/distance_metrics/cosine_distance.py @@ -0,0 +1,65 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: +Summary +------- +Module for the ZnTrack cosine distance. +""" +import jax.numpy as np + +from symsuite.distance_metrics.distance_metric import DistanceMetric + + +class CosineDistance(DistanceMetric): + """ + Class for the cosine distance metric. + + Notes + ----- + This is not a real distance metric. + """ + + def __call__(self, point_1: np.ndarray, point_2: np.ndarray, **kwargs): + """ + Call the distance metric. + + Distance between points in the point_1 tensor will be computed between those in + the point_2 tensor element-wise. Therefore, we will have: + + point_1[i] - point_2[i] for all i. + + Parameters + ---------- + point_1 : np.ndarray (n_points, point_dimension) + First set of points in the comparison. + point_2 : np.ndarray (n_points, point_dimension) + Second set of points in the comparison. + kwargs + Miscellaneous keyword arguments for the specific metric. + + Returns + ------- + d(point_1, point_2) : tf.tensor : shape=(n_points, 1) + Array of distances for each point. + """ + numerator = np.einsum("ij, ij -> i", point_1, point_2) + denominator = np.sqrt( + np.einsum("ij, ij -> i", point_1, point_1) + * np.einsum("ij, ij -> i", point_2, point_2) + ) + + return 1 - abs(np.divide(numerator, denominator)) diff --git a/symsuite/distance_metrics/distance_metric.py b/symsuite/distance_metrics/distance_metric.py new file mode 100644 index 0000000..49761a6 --- /dev/null +++ b/symsuite/distance_metrics/distance_metric.py @@ -0,0 +1,54 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Module for the parent class of a ZnRND distance metric. +""" +import jax.numpy as np + + +class DistanceMetric: + """ + Parent class for a ZnRND distance metric. + """ + + def __call__(self, point_1: np.ndarray, point_2: np.ndarray, **kwargs): + """ + Call the distance metric. + + Distance between points in the point_1 tensor will be computed between those in + the point_2 tensor element-wise. Therefore, we will have: + + point_1[i] - point_2[i] for all i. + + Parameters + ---------- + point_1 : np.ndarray (n_points, point_dimension) + First set of points in the comparison. + point_2 : np.ndarray (n_points, point_dimension) + Second set of points in the comparison. + kwargs + Miscellaneous keyword arguments for the specific metric. + + Returns + ------- + d(point_1, point_2) : tf.tensor : shape=(n_points, 1) + Array of distances for each point. + """ + raise NotImplementedError("Implemented in child class.") diff --git a/symsuite/distance_metrics/hyper_sphere_distance.py b/symsuite/distance_metrics/hyper_sphere_distance.py new file mode 100644 index 0000000..2a1aa31 --- /dev/null +++ b/symsuite/distance_metrics/hyper_sphere_distance.py @@ -0,0 +1,71 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Module for a distance that combines the properties of cosine and lp-norm distance. +""" +import jax.numpy as np + +from symsuite.distance_metrics.cosine_distance import CosineDistance +from symsuite.distance_metrics.distance_metric import DistanceMetric +from symsuite.distance_metrics.l_p_norm import LPNorm + + +class HyperSphere(DistanceMetric): + """ + Compute the L_p norm between vectors. + """ + + def __init__(self, order: float): + """ + Constructor for the LPNorm class. + + Parameters + ---------- + order : float + order of the space + """ + self.order = order + + def __call__(self, point_1: np.ndarray, point_2: np.ndarray, **kwargs): + """ + Call the distance metric. + + Distance between points in the point_1 tensor will be computed between those in + the point_2 tensor element-wise. Therefore, we will have: + + point_1[i] - point_2[i] for all i. + + Parameters + ---------- + point_1 : np.ndarray (n_points, point_dimension) + First set of points in the comparison. + point_2 : np.ndarray (n_points, point_dimension) + Second set of points in the comparison. + kwargs + Miscellaneous keyword arguments for the specific metric. + + Returns + ------- + d(point_1, point_2) : tf.tensor : shape=(n_points, 1) + Array of distances for each point. + """ + return LPNorm(order=self.order)(point_1, point_2) * CosineDistance()( + point_1, point_2 + ) diff --git a/symsuite/distance_metrics/l_p_norm.py b/symsuite/distance_metrics/l_p_norm.py new file mode 100644 index 0000000..9925644 --- /dev/null +++ b/symsuite/distance_metrics/l_p_norm.py @@ -0,0 +1,71 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Module for the L_p norm class. + + r = p1 - p2 + + d = (|r[0]|^p + |r[1]|^p + ... + |r[n]|^p)^(1/p) +""" +import jax.numpy as np + +from symsuite.distance_metrics.distance_metric import DistanceMetric + + +class LPNorm(DistanceMetric): + """ + Compute the L_p norm between vectors. + """ + + def __init__(self, order: float): + """ + Constructor for the LPNorm class. + + Parameters + ---------- + order : float + order of the space + """ + self.order = order + + def __call__(self, point_1: np.ndarray, point_2: np.ndarray, **kwargs): + """ + Call the distance metric. + + Distance between points in the point_1 tensor will be computed between those in + the point_2 tensor element-wise. Therefore, we will have: + + point_1[i] - point_2[i] for all i. + + Parameters + ---------- + point_1 : np.ndarray (n_points, point_dimension) + First set of points in the comparison. + point_2 : np.ndarray (n_points, point_dimension) + Second set of points in the comparison. + kwargs + Miscellaneous keyword arguments for the specific metric. + + Returns + ------- + d(point_1, point_2) : np.ndarray : shape=(n_points, 1) + Array of distances for each point. + """ + return np.linalg.norm(point_1 - point_2, axis=1, ord=self.order) diff --git a/symsuite/distance_metrics/mahalanobis_distance.py b/symsuite/distance_metrics/mahalanobis_distance.py new file mode 100644 index 0000000..b1ef019 --- /dev/null +++ b/symsuite/distance_metrics/mahalanobis_distance.py @@ -0,0 +1,67 @@ +""" +ZnRND: A Zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Module for the Mahalanobis distance. +""" +import jax.numpy as np +import scipy.spatial.distance + +from symsuite.distance_metrics.distance_metric import DistanceMetric + + +class MahalanobisDistance(DistanceMetric): + """ + Compute the mahalanobis distance between points. + """ + + def __call__(self, point_1: np.array, point_2: np.array, **kwargs) -> np.array: + """ + Call the distance metric. + + Mahalanobis Distance between points in the point_1 tensor will be computed + between those in the point_2 tensor element-wise. Therefore, we will have: + + point_1[i] - point_2[i] for all i. + + Parameters + ---------- + point_1 : tf.Tensor (n_points, point_dimension) + First set of points in the comparison. + point_2 : tf.Tensor (n_points, point_dimension) + Second set of points in the comparison. + kwargs + Miscellaneous keyword arguments for the specific metric. + Returns + ------- + d(point_1, point_2) : tf.tensor : shape=(n_points, 1) + Array of distances for each point. + """ + inverted_covariance = np.linalg.inv(np.cov(point_1.T)) + distances = [] + for i in range(len(point_1.T[0, :])): + distance = scipy.spatial.distance.mahalanobis( + point_1[i], point_2[i], inverted_covariance + ) + distances.append(distance) + + return distances diff --git a/symsuite/distance_metrics/order_n_difference.py b/symsuite/distance_metrics/order_n_difference.py new file mode 100644 index 0000000..c36e96a --- /dev/null +++ b/symsuite/distance_metrics/order_n_difference.py @@ -0,0 +1,79 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Raise a difference to a power of order n. + +e.g. (a - b)^n +""" +import jax.numpy as np + +from symsuite.distance_metrics.distance_metric import DistanceMetric + + +class OrderNDifference(DistanceMetric): + """ + Compute the order n difference between points. + """ + + def __init__(self, order: float = 2, reduce_operation: str = "mean"): + """ + Constructor for the order n distance. + + Parameters + ---------- + order : float (default=2) + Order to which the difference should be raised. + reduce_operation : str (default = "mean") + How to reduce the order N difference, either a sum or a mean. + """ + self.order = order + self.reduce_operation = reduce_operation + + def __call__(self, point_1: np.ndarray, point_2: np.ndarray, **kwargs): + """ + Call the distance metric. + + Distance between points in the point_1 tensor will be computed between those in + the point_2 tensor element-wise. Therefore, we will have: + + point_1[i] - point_2[i] for all i. + + Parameters + ---------- + point_1 : np.ndarray (n_points, point_dimension) + First set of points in the comparison. + point_2 : np.ndarray (n_points, point_dimension) + Second set of points in the comparison. + kwargs + Miscellaneous keyword arguments for the specific metric. + + Returns + ------- + d(point_1, point_2) : np.ndarray : shape=(n_points, 1) + Array of distances for each point. + """ + diff = point_1 - point_2 + + if self.reduce_operation == "mean": + return np.mean(np.power(diff, self.order), axis=1) + elif self.reduce_operation == "sum": + return np.sum(np.power(diff, self.order), axis=1) + else: + raise ValueError(f"Invalid reduction operation: {self.reduce_operation}") diff --git a/symsuite/generator_extraction/generators.py b/symsuite/generator_extraction/generators.py index 60bb02f..239f6cb 100644 --- a/symsuite/generator_extraction/generators.py +++ b/symsuite/generator_extraction/generators.py @@ -10,15 +10,16 @@ ========== Python module to extract generators from data. """ -import tensorflow as tf -import numpy as np -from tqdm import tqdm import random -from sklearn.linear_model import LinearRegression -from sklearn.decomposition import PCA -import matplotlib.pyplot as plt from typing import Tuple +import jax.numpy as jnp +import matplotlib.pyplot as plt +import numpy as np +from sklearn.decomposition import PCA +from sklearn.linear_model import LinearRegression +from tqdm import tqdm + class GeneratorExtraction: """ @@ -52,7 +53,7 @@ class GeneratorExtraction: def __init__( self, - point_cloud: tf.Tensor, + point_cloud: jnp.array, delta: float = 0.5, epsilon: float = 0.3, candidate_runs: int = 10, @@ -77,8 +78,8 @@ def __init__( self.epsilon = epsilon self.candidate_runs = candidate_runs - self.basis: tf.Tensor - self.hyperplane_set: tf.Tensor + self.basis: jnp.ndarray + self.hyperplane_set: jnp.ndarray self.point_pairs: list self.dimension = self._get_dimension() @@ -134,7 +135,7 @@ def _generate_basis_set(self, gs_precision: int): for item in reduced_candidates: basis.append(self._perform_gs(item, basis)) - self.basis = tf.convert_to_tensor(basis) # set the class attribute + self.basis = jnp.array(basis) # set the class attribute self._gs_check(gs_precision) def _gs_check(self, gs_precision: int): @@ -153,11 +154,14 @@ def _gs_check(self, gs_precision: int): Will throw an exception if the assert fails. """ for basis in self.basis: - np.testing.assert_almost_equal(np.linalg.norm(basis), 1) # check the normalization. + # check the normalization. + np.testing.assert_almost_equal(np.linalg.norm(basis), 1) for test in self.basis: if all(test == basis): continue - np.testing.assert_almost_equal(np.dot(basis, test), 0, decimal=gs_precision) # check the orthogonality. + np.testing.assert_almost_equal( + np.dot(basis, test), 0, decimal=gs_precision + ) # check the orthogonality. def _perform_gs(self, vector: list, basis_set: list) -> np.ndarray: """ @@ -178,12 +182,11 @@ def _perform_gs(self, vector: list, basis_set: list) -> np.ndarray: for basis_item in basis_set: basis_vector -= self._projection_operator(basis_item, basis_vector) - return basis_vector / np.linalg.norm(basis_vector) + return jnp.array(basis_vector) / np.linalg.norm(basis_vector) def _eliminate_closest_vector( - self, - reference_vectors: list, - test_vectors: list) -> np.ndarray: + self, reference_vectors: list, test_vectors: list + ) -> np.ndarray: """ Remove the closest vectors in the theoretical basis set @@ -258,7 +261,7 @@ def _construct_hyperplane_set(self): if all(truth_table): self.hyperplane_set.append(point) - self.hyperplane_set = tf.convert_to_tensor(self.hyperplane_set) + self.hyperplane_set = jnp.array(self.hyperplane_set) def _identify_point_pairs(self): """ @@ -336,28 +339,31 @@ def _full_regression(self): def _simple_regression(self): """ - In the case where additional constraints are not needed, we simply perform regression on the problem to - extract generator candidates. + In the case where additional constraints are not needed, we simply perform + regression on the problem to extract generator candidates. Returns ------- Updates the class state. """ - Y = [] - X = [] + y_data = [] + x_data = [] for pair in self.point_pairs: points = [self.hyperplane_set[pair[0]], self.hyperplane_set[pair[1]]] sigma = self._compute_sigma(points) - Y.append( + y_data.append( ((points[0] - points[1]) * np.linalg.norm(points[0])) / (sigma * np.linalg.norm(points[1] - points[0])) ) - X.append(points[0]) + x_data.append(points[0]) generator = [] for i in range(self.dimension): generator = np.concatenate( - (generator, LinearRegression().fit(X, np.array(Y)[:, i]).coef_) + ( + generator, + LinearRegression().fit(x_data, np.array(y_data)[:, i]).coef_, + ) ) self.generator_candidates.append(generator) @@ -381,7 +387,7 @@ def _compute_sigma(self, pair) -> int: ) def _extract_generators( - self, pca_components: object, factor: object = True + self, pca_components: object, factor: object = True ) -> tuple: """ Perform PCA on candidates and extract true generators. @@ -407,9 +413,12 @@ def _extract_generators( pca = PCA(n_components=pca_components) pca.fit(self.generator_candidates) if factor: - return np.sqrt(self.dimension) * pca.components_, pca.explained_variance_ratio_ + return ( + np.sqrt(self.dimension) * pca.components_, + pca.explained_variance_ratio_, + ) else: - return (pca.components_, pca.explained_variance_ratio_) + return pca.components_, pca.explained_variance_ratio_ def _plot_results(self, std_values: list, save: bool = False): """ @@ -438,12 +447,13 @@ def _plot_results(self, std_values: list, save: bool = False): plt.show() def perform_generator_extraction( - self, - pca_components: int = 4, - plot: bool = False, - save: bool = False, - factor: bool = True, - gs_precision: int = 5) -> Tuple: + self, + pca_components: int = 4, + plot: bool = False, + save: bool = False, + factor: bool = True, + gs_precision: int = 5, + ) -> Tuple: """ Collect all methods and perform the generator extraction. @@ -468,9 +478,9 @@ def perform_generator_extraction( explained variance list. """ - for _ in tqdm(range(self.candidate_runs), - ncols=100, - desc="Producing generator candidates"): + for _ in tqdm( + range(self.candidate_runs), ncols=100, desc="Producing generator candidates" + ): try: self._remove_redundancy() self._generate_basis_set(gs_precision) @@ -480,8 +490,9 @@ def perform_generator_extraction( except ValueError: continue - generators, std_array = self._extract_generators(pca_components=pca_components, - factor=factor) + generators, std_array = self._extract_generators( + pca_components=pca_components, factor=factor + ) for i, item in enumerate(generators): print(f"Principle Component {i + 1}: Explained Variance: {std_array[i]}") print(item.reshape((self.dimension, self.dimension))) diff --git a/symsuite/loss_functions/__init__.py b/symsuite/loss_functions/__init__.py new file mode 100644 index 0000000..2460dca --- /dev/null +++ b/symsuite/loss_functions/__init__.py @@ -0,0 +1,44 @@ +""" +Symsuite + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Package containing custom loss functions. +""" +from symsuite.loss_functions.absolute_angle_difference import AngleDistanceLoss +from symsuite.loss_functions.cosine_distance import CosineDistanceLoss +from symsuite.loss_functions.cross_entropy_loss import CrossEntropyLoss +from symsuite.loss_functions.l_p_norm import LPNormLoss +from symsuite.loss_functions.mahalanobis import MahalanobisLoss +from symsuite.loss_functions.mean_power_error import MeanPowerLoss +from symsuite.loss_functions.loss import Loss + +__all__ = [ + AngleDistanceLoss.__name__, + CosineDistanceLoss.__name__, + LPNormLoss.__name__, + MahalanobisLoss.__name__, + MeanPowerLoss.__name__, + Loss.__name__, + CrossEntropyLoss.__name__, +] diff --git a/symsuite/loss_functions/absolute_angle_difference.py b/symsuite/loss_functions/absolute_angle_difference.py new file mode 100644 index 0000000..31d6df5 --- /dev/null +++ b/symsuite/loss_functions/absolute_angle_difference.py @@ -0,0 +1,37 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +ZnRND absolute angle difference TF loss function. +""" +from symsuite.distance_metrics.angular_distance import AngularDistance +from symsuite.loss_functions.loss import Loss + + +class AngleDistanceLoss(Loss): + """ + Class for the mean power loss + """ + + def __init__(self): + """ + Constructor for the mean power loss class. + """ + super(AngleDistanceLoss, self).__init__() + self.metric = AngularDistance() diff --git a/symsuite/loss_functions/cosine_distance.py b/symsuite/loss_functions/cosine_distance.py new file mode 100644 index 0000000..73e157c --- /dev/null +++ b/symsuite/loss_functions/cosine_distance.py @@ -0,0 +1,37 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +ZnRND Cosine similarity TF loss function. +""" +from symsuite.distance_metrics.cosine_distance import CosineDistance +from symsuite.loss_functions.loss import Loss + + +class CosineDistanceLoss(Loss): + """ + Class for the mean power loss + """ + + def __init__(self): + """ + Constructor for the mean power loss class. + """ + super(CosineDistanceLoss, self).__init__() + self.metric = CosineDistance() diff --git a/symsuite/loss_functions/cross_entropy_loss.py b/symsuite/loss_functions/cross_entropy_loss.py new file mode 100644 index 0000000..a21bccf --- /dev/null +++ b/symsuite/loss_functions/cross_entropy_loss.py @@ -0,0 +1,88 @@ +""" +ZnRND: A zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Implement a cross entropy loss function. +""" +import jax +import optax + +from symsuite.loss_functions.loss import Loss + + +class CrossEntropyDistance: + """ + Class for the cross entropy distance + """ + + def __init__(self, classes: int, apply_softmax: bool = False): + """ + Constructor for the distance + + Parameters + ---------- + classes : int + Number of classes in the one-hot encoding. + apply_softmax : bool (default = False) + If true, softmax is applied to the prediction before computing the loss. + """ + self.classes = classes + self.apply_softmax = apply_softmax + + def __call__(self, prediction, target): + """ + + Parameters + ---------- + prediction (batch_size, n_classes) + target + + Returns + ------- + + """ + if self.apply_softmax: + prediction = jax.nn.softmax(prediction) + one_hot_labels = jax.nn.one_hot(target, num_classes=self.classes) + return optax.softmax_cross_entropy(logits=prediction, labels=one_hot_labels) + + +class CrossEntropyLoss(Loss): + """ + Class for the cross entropy loss + """ + + def __init__(self, classes: int = 10, apply_softmax: bool = False): + """ + Constructor for the mean power loss class. + + Parameters + ---------- + classes : int (default=10) + Number of classes in the loss. + apply_softmax : bool (default = False) + If true, softmax is applied to the prediction before computing the loss. + """ + super(CrossEntropyLoss, self).__init__() + self.metric = CrossEntropyDistance(classes=classes, apply_softmax=apply_softmax) diff --git a/symsuite/loss_functions/l_p_norm.py b/symsuite/loss_functions/l_p_norm.py new file mode 100644 index 0000000..39616bd --- /dev/null +++ b/symsuite/loss_functions/l_p_norm.py @@ -0,0 +1,42 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +ZnRND L^{p} norm TF loss function. +""" +from symsuite.distance_metrics.l_p_norm import LPNorm +from symsuite.loss_functions.loss import Loss + + +class LPNormLoss(Loss): + """ + Class for the mean power loss + """ + + def __init__(self, order: float): + """ + Constructor for the L_p norm loss class. + + Parameters + ---------- + order : float + Order to which the difference should be raised. + """ + super(LPNormLoss, self).__init__() + self.metric = LPNorm(order=order) diff --git a/symsuite/loss_functions/loss.py b/symsuite/loss_functions/loss.py new file mode 100644 index 0000000..d19c474 --- /dev/null +++ b/symsuite/loss_functions/loss.py @@ -0,0 +1,64 @@ +""" +ZnRND: A Zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Module for the simple loss for TensorFlow. +""" +from abc import ABC + +import jax.numpy as np + +from symsuite.distance_metrics.distance_metric import DistanceMetric + + +class Loss(ABC): + """ + Class for the simple loss. + + Attributes + ---------- + metric : DistanceMetric + """ + + def __init__(self): + """ + Constructor for the simple loss parent class. + """ + super().__init__() + self.metric: DistanceMetric = None + + def __call__(self, point_1: np.array, point_2: np.array) -> float: + """ + Summation over the tensor of the respective similarity measurement + Parameters + ---------- + point_1 : np.array + first neural network representation of the considered points + point_2 : np.array + second neural network representation of the considered points + + Returns + ------- + loss : float + total loss of all points based on the similarity measurement + """ + return np.mean(self.metric(point_1, point_2), axis=0) diff --git a/symsuite/loss_functions/mahalanobis.py b/symsuite/loss_functions/mahalanobis.py new file mode 100644 index 0000000..d16e4eb --- /dev/null +++ b/symsuite/loss_functions/mahalanobis.py @@ -0,0 +1,37 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +ZnRND Mahalanobis distance TF loss function. +""" +import symsuite.distance_metrics.mahalanobis_distance as mahalanobis +from symsuite.loss_functions.loss import Loss + + +class MahalanobisLoss(Loss): + """ + Class for the mean power loss + """ + + def __init__(self): + """ + Constructor for the Mahalanobis loss class. + """ + super(MahalanobisLoss, self).__init__() + self.metric = mahalanobis.MahalanobisDistance() diff --git a/symsuite/loss_functions/mean_power_error.py b/symsuite/loss_functions/mean_power_error.py new file mode 100644 index 0000000..4700182 --- /dev/null +++ b/symsuite/loss_functions/mean_power_error.py @@ -0,0 +1,42 @@ +""" +ZnRND: A Zincwarecode package. +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 +Copyright Contributors to the Zincwarecode Project. +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ +Citation +-------- +If you use this module please cite us with: + +Summary +------- +ZnRND mean square error TF loss function. +""" +from symsuite.distance_metrics.order_n_difference import OrderNDifference +from symsuite.loss_functions.loss import Loss + + +class MeanPowerLoss(Loss): + """ + Class for the mean power loss + """ + + def __init__(self, order: float): + """ + Constructor for the mean power loss class. + + Parameters + ---------- + order : float + Order to which the difference should be raised. + """ + super(MeanPowerLoss, self).__init__() + self.metric = OrderNDifference(order=order) diff --git a/symsuite/ml_models/dense_model.py b/symsuite/ml_models/dense_model.py index deba5f0..1e33e21 100644 --- a/symsuite/ml_models/dense_model.py +++ b/symsuite/ml_models/dense_model.py @@ -10,8 +10,8 @@ =========== Dense neural network model """ -import tensorflow as tf import numpy as np +import tensorflow as tf from tensorflow.keras import regularizers from tensorflow.keras.layers import Input @@ -23,8 +23,9 @@ class DenseModel: Attributes ---------- data_dict : dict - Dictionary of data where the key is the class name and the values are the coordinates belongin to that - class. This is fundamentall a classification problem! + Dictionary of data where the key is the class name and the values are the + coordinates belonging to that class. This is fundamental to a classification + problem. n_layers : int Number of hidden layers to use. units : int @@ -63,7 +64,8 @@ def __init__( lr: float = 1e-4, batch_size: int = 100, terminate_patience: int = 10, - lr_patience: int = 5): + lr_patience: int = 5, + ): """ Constructor fpr the Dense model class. @@ -121,7 +123,9 @@ def add_data(self, data: dict): """ self.data_dict = data - self.input_shape = len(self.data_dict[list(self.data_dict.keys())[0]]['domain'][0]) + self.input_shape = len( + self.data_dict[list(self.data_dict.keys())[0]]["domain"][0] + ) def _shuffle_and_split_data(self): """ @@ -135,12 +139,16 @@ def _shuffle_and_split_data(self): for key in self.data_dict: labels = tf.repeat( tf.convert_to_tensor(np.array(key), dtype=tf.float32), - len(self.data_dict[key]['domain']), + len(self.data_dict[key]["domain"]), ) - stacked_data = tf.concat([tf.cast(self.data_dict[key]['domain'], dtype=tf.float32), - tf.transpose([labels])], - axis=1) + stacked_data = tf.concat( + [ + tf.cast(self.data_dict[key]["domain"], dtype=tf.float32), + tf.transpose([labels]), + ], + axis=1, + ) data_volume = len(stacked_data) train, test, validate = tf.split( @@ -155,18 +163,15 @@ def _shuffle_and_split_data(self): if self.train_ds is None: self.train_ds = train else: - self.train_ds = tf.concat([self.train_ds, train], - axis=0) + self.train_ds = tf.concat([self.train_ds, train], axis=0) if self.test_ds is None: self.test_ds = test else: - self.test_ds = tf.concat([self.test_ds, test], - axis=0) + self.test_ds = tf.concat([self.test_ds, test], axis=0) if self.val_ds is None: self.val_ds = validate else: - self.val_ds = tf.concat([self.val_ds, validate], - axis=0) + self.val_ds = tf.concat([self.val_ds, validate], axis=0) self.train_ds = tf.random.shuffle(self.train_ds) self.test_ds = tf.random.shuffle(self.test_ds) @@ -268,12 +273,12 @@ def train_model(self): # Train the model for i in range(1, 6): self.model.fit( - x=self.train_ds[:, 0:self.input_shape], + x=self.train_ds[:, 0 : self.input_shape], y=tf.keras.utils.to_categorical(self.train_ds[:, -1]), batch_size=self.batch_size, shuffle=True, validation_data=( - self.test_ds[:, 0:self.input_shape], + self.test_ds[:, 0 : self.input_shape], tf.keras.utils.to_categorical(self.test_ds[:, -1]), ), verbose=1, @@ -290,7 +295,8 @@ def _evaluate_model(self): """ attributes = self.model.evaluate( - x=self.val_ds[:, 0:self.input_shape], y=tf.keras.utils.to_categorical(self.val_ds[:, -1]) + x=self.val_ds[:, 0 : self.input_shape], + y=tf.keras.utils.to_categorical(self.val_ds[:, -1]), ) print(f"Loss: {attributes[0]} \n" f"Accuracy: {attributes[1]}") @@ -316,4 +322,4 @@ def get_embedding_layer_representation(self, data_array: np.ndarray) -> tf.Tenso model.add(layer) model.build() - return model.predict(data_array[:, 0:self.input_shape]) + return model.predict(data_array[:, 0 : self.input_shape]) diff --git a/symsuite/ml_models/model.py b/symsuite/ml_models/model.py new file mode 100644 index 0000000..4eb9221 --- /dev/null +++ b/symsuite/ml_models/model.py @@ -0,0 +1,118 @@ +""" +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the Zincware Project. + +Description: Parent class for the models. +""" +from typing import Any, Callable, Union + +import jax.numpy as jnp +from jax.random import PRNGKeyArray + + +class Model: + """ + Parent class for ZnRND Models. + + Attributes + ---------- + model : Callable + A callable class or function that takes a feature + vector and returns something from it. Typically this is a + neural network layer stack. + """ + + model: Callable + + def init_model( + self, + init_rng: Union[Any, PRNGKeyArray] = None, + kernel_init: Callable = None, + bias_init: Callable = None, + ): + """ + Initialize a model. + + Parameters + ---------- + init_rng : Union[Any, PRNGKeyArray] + Initial rng for train state that is immediately deleted. + kernel_init : Callable + Define the kernel initialization. + bias_init : Callable + Define the bias initialization. + """ + raise NotImplementedError("Implemented in child class.") + + def train_model( + self, + train_ds: dict, + test_ds: dict, + epochs: int = 10, + batch_size: int = 1, + disable_loading_bar: bool = False, + ): + """ + Train the model on data. + + Parameters + ---------- + train_ds : dict + Train dataset with inputs and targets. + test_ds : dict + Test dataset with inputs and targets. + epochs : int + Number of epochs to train over. + batch_size : int + Size of the batch to use in training. + disable_loading_bar : bool + Disable the output visualization of the loading par. + """ + raise NotImplementedError("Implemented in child class.") + + def compute_ntk( + self, + x_i: jnp.ndarray, + x_j: jnp.ndarray = None, + normalize: bool = True, + infinite: bool = False, + ): + """ + Compute the NTK matrix for the model. + + Parameters + ---------- + x_i : jnp.ndarray + Dataset for which to compute the NTK matrix. + x_j : jnp.ndarray (optional) + Dataset for which to compute the NTK matrix. + normalize : bool (default = True) + If true, divide each row by its max value. + infinite : bool (default = False) + If true, compute the infinite width limit as well. + + Returns + ------- + NTK : dict + The NTK matrix for both the empirical and infinite width computation. + """ + raise NotImplementedError("Implemented in child class") + + def __call__(self, feature_vector: jnp.ndarray): + """ + Call the network. + + Parameters + ---------- + feature_vector : jnp.ndarray + Feature vector on which to apply operation. + + Returns + ------- + output of the model. + """ + self.model(feature_vector) diff --git a/symsuite/ml_models/nt_model.py b/symsuite/ml_models/nt_model.py new file mode 100644 index 0000000..a6cb84c --- /dev/null +++ b/symsuite/ml_models/nt_model.py @@ -0,0 +1,459 @@ +""" +ZnRND: A zincwarecode package. + +License +------- +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v2.0 which accompanies this distribution, and is +available at https://www.eclipse.org/legal/epl-v20.html + +SPDX-License-Identifier: EPL-2.0 + +Copyright Contributors to the zincwarecode Project. + +Contact Information +------------------- +email: zincwarecode@gmail.com +github: https://github.com/zincware +web: https://zincwarecode.com/ + +Citation +-------- +If you use this module please cite us with: + +Summary +------- +Module for the neural tangents infinite width network models. +""" +import logging +from typing import Any, Callable, Union + +import jax +import jax.numpy as jnp +import neural_tangents as nt +import numpy as onp +from flax.training import train_state +from jax.random import PRNGKeyArray +from neural_tangents.stax import serial +from tqdm import trange + +from symsuite.accuracy_functions.accuracy_function import AccuracyFunction +from symsuite.loss_functions.loss import Loss +from symsuite.ml_models.model import Model +from symsuite.utils import normalize_covariance_matrix + +logger = logging.getLogger(__name__) + + +class NTModel(Model): + """ + Class for a neural tangents model. + """ + + def __init__( + self, + loss_fn: Loss, + optimizer: Callable, + input_shape: tuple, + nt_module: serial = None, + data_pool: jnp.ndarray = None, + accuracy_fn: AccuracyFunction = None, + batch_size: int = 10, + ): + """ + Constructor for a Flax model. + + Parameters + ---------- + loss_fn : SimpleLoss + A function to use in the loss computation. + accuracy_fn : AccuracyFunction + Accuracy function to use for accuracy computation. + optimizer : Callable + optimizer to use in the training. OpTax is used by default and + cross-compatibility is not assured. + input_shape : tuple + Shape of the NN input. + batch_size : int (default=10) + Batch size to use in the NTK computation. + nt_module : serial + NT stax module for training. + data_pool : jnp.ndarray + Data pool from which TTV is built. + + """ + self.rng = jax.random.PRNGKey(onp.random.randint(0, 500)) + self.init_fn = nt_module[0] + self.apply_fn = jax.jit(nt_module[1]) + self.kernel_fn = nt.batch(nt_module[2], batch_size=batch_size) + self.empirical_ntk = nt.batch( + nt.empirical_ntk_fn(self.apply_fn), batch_size=batch_size + ) + self.empirical_ntk_jit = jax.jit(self.empirical_ntk) + self.loss_fn = loss_fn + self.accuracy_fn = accuracy_fn + self.optimizer = optimizer + self.input_shape = input_shape + + self.data_pool = data_pool + + # initialize the model state + self.model_state = None + self.init_model() + + def init_model( + self, + init_rng: Union[Any, PRNGKeyArray] = None, + kernel_init: Callable = None, + bias_init: Callable = None, + ): + """ + Initialize a model. + + If no rng key is given, the key will be produced randomly. + + Parameters + ---------- + init_rng : Union[Any, PRNGKeyArray] + Initial rng for train state that is immediately deleted. + kernel_init : Callable + Define the kernel initialization. + bias_init : Callable + Define the bias initialization. + """ + if kernel_init: + raise NotImplementedError( + "Currently, there is no option customize the weight initialization. " + ) + if bias_init: + raise NotImplementedError( + "Currently, there is no option customize the bias initialization. " + ) + if init_rng is None: + init_rng = jax.random.PRNGKey(onp.random.randint(0, 1000000)) + self.model_state = self._create_train_state(init_rng) + + def compute_ntk( + self, + x_i: jnp.ndarray, + x_j: jnp.ndarray = None, + normalize: bool = True, + infinite: bool = False, + ): + """ + Compute the NTK matrix for the model. + + Parameters + ---------- + x_i : np.ndarray + Dataset for which to compute the NTK matrix. + x_j : np.ndarray (optional) + Dataset for which to compute the NTK matrix. + normalize : bool (default = True) + If true, divide each row by its max value. + infinite : bool (default = False) + If true, compute the infinite width limit as well. + + Returns + ------- + NTK : dict + The NTK matrix for both the empirical and infinite width computation. + """ + if x_j is None: + x_j = x_i + empirical_ntk = self.empirical_ntk_jit(x_i, x_j, self.model_state.params) + + if infinite: + infinite_ntk = self.kernel_fn(x_i, x_j, "ntk") + else: + infinite_ntk = None + + if normalize: + empirical_ntk = normalize_covariance_matrix(empirical_ntk) + if infinite: + infinite_ntk = normalize_covariance_matrix(infinite_ntk) + + return {"empirical": empirical_ntk, "infinite": infinite_ntk} + + def _create_train_state(self, init_rng: Union[Any, PRNGKeyArray]): + """ + Create a training state of the model. + + Parameters + ---------- + init_rng : Union[Any, PRNGKeyArray] + Initial rng for train state that is immediately deleted. + + Returns + ------- + initial state of model to then be trained. + + Notes + ----- + TODO: Make the TrainState class passable by the user as it can track custom + model properties. + """ + _, params = self.init_fn(init_rng, self.input_shape) + + return train_state.TrainState.create( + apply_fn=self.apply_fn, params=params, tx=self.optimizer + ) + + def _compute_metrics( + self, + predictions: jnp.ndarray, + targets: jnp.ndarray, + ): + """ + Compute the current metrics of the training. + + Parameters + ---------- + predictions : np.ndarray + Predictions made by the network. + targets : np.ndarray + Targets from the training data. + + Returns + ------- + metrics : dict + A dict of current training metrics, e.g. {"loss": ..., "accuracy": ...} + """ + loss = self.loss_fn(predictions, targets) + if self.accuracy_fn is not None: + accuracy = self.accuracy_fn(predictions, targets) + metrics = {"loss": loss, "accuracy": accuracy} + + else: + metrics = {"loss": loss} + + return metrics + + def _train_step(self, state: train_state.TrainState, batch: dict): + """ + Train a single step. + + Parameters + ---------- + state : TrainState + Current state of the neural network. + batch : dict + Batch of data to train on. + + Returns + ------- + state : dict + Updated state of the neural network. + metrics : dict + Metrics for the current model. + """ + + def loss_fn(params): + """ + helper loss computation + """ + inner_predictions = self.apply_fn(params, batch["inputs"]) + loss = self.loss_fn(inner_predictions, batch["targets"]) + return loss, inner_predictions + + grad_fn = jax.value_and_grad(loss_fn, has_aux=True) + + (_, predictions), grads = grad_fn(state.params) + + state = state.apply_gradients(grads=grads) # in place state update. + metrics = self._compute_metrics( + predictions=predictions, targets=batch["targets"] + ) + + return state, metrics + + def _evaluate_step(self, params: dict, batch: dict): + """ + Evaluate the model on test data. + + Parameters + ---------- + params : dict + Parameters of the model. + batch : dict + Batch of data to test on. + + Returns + ------- + metrics : dict + Metrics dict computed on test data. + """ + predictions = self.apply_fn(params, batch["inputs"]) + + return self._compute_metrics(predictions, batch["targets"]) + + def _train_epoch( + self, state: train_state.TrainState, train_ds: dict, batch_size: int + ): + """ + Train for a single epoch. + + Performs the following steps: + + * Shuffles the data + * Runs an optimization step on each batch + * Computes the metrics for the batch + * Return an updated optimizer, state, and metrics dictionary. + + Parameters + ---------- + state : TrainState + Current state of the model. + train_ds : dict + Dataset on which to train. + batch_size : int + Size of each batch. + + Returns + ------- + state : TrainState + State of the model after the epoch. + metrics : dict + Dict of metrics for current state. + """ + # Some housekeeping variables. + train_ds_size = len(train_ds["inputs"]) + steps_per_epoch = train_ds_size // batch_size + + if train_ds_size == 1: + state, metrics = self._train_step(state, train_ds) + batch_metrics = [metrics] + + else: + # Prepare the shuffle. + permutations = jax.random.permutation(self.rng, train_ds_size) + permutations = permutations[: steps_per_epoch * batch_size] + permutations = permutations.reshape((steps_per_epoch, batch_size)) + + # Step over items in batch. + batch_metrics = [] + for permutation in permutations: + batch = {k: v[permutation, ...] for k, v in train_ds.items()} + # print(batch) + state, metrics = self._train_step(state, batch) + batch_metrics.append(metrics) + + # Get the metrics off device for printing. + batch_metrics_np = jax.device_get(batch_metrics) + epoch_metrics_np = { + k: onp.mean([metrics[k] for metrics in batch_metrics_np]) + for k in batch_metrics_np[0] + } + + return state, epoch_metrics_np + + def _evaluate_model(self, params: dict, test_ds: dict) -> dict: + """ + Evaluate the model. + + Parameters + ---------- + params : dict + Current state of the model. + test_ds : dict + Dataset on which to evaluate. + Returns + ------- + metrics : dict + Loss of the model. + """ + metrics = self._evaluate_step(params, test_ds) + metrics = jax.device_get(metrics) + summary = jax.tree_map(lambda x: x.item(), metrics) + + return summary + + def validate_model(self, dataset: dict, loss_fn: SimpleLoss): + """ + Validate the model on some external data. + + Parameters + ---------- + loss_fn : SimpleLoss + Loss function to use in the computation. + dataset : dict + Dataset on which to validate the model. + {"inputs": np.ndarray, "targets": np.ndarray}. + + Returns + ------- + metrics : dict + Metrics computed in the validation. {"loss": [], "accuracy": []}. + Note, for ease of large scale experiments we always return both keywords + whether they are computed or not. + """ + predictions = self.apply_fn(self.model_state.params, dataset["inputs"]) + + loss = loss_fn(predictions, dataset["targets"]) + + if self.accuracy_fn is not None: + accuracy = self.accuracy_fn(predictions, dataset["targets"]) + else: + accuracy = None + + return {"loss": loss, "accuracy": accuracy} + + def train_model( + self, + train_ds: dict, + test_ds: dict, + epochs: int = 50, + batch_size: int = 1, + disable_loading_bar: bool = False, + ): + """ + Train the model. + + See the parent class for a full doc-string. + """ + if self.model_state is None: + self.init_model() + + state = self.model_state + + loading_bar = trange( + 1, epochs + 1, ncols=100, unit="batch", disable=disable_loading_bar + ) + test_losses = [] + test_accuracy = [] + train_losses = [] + train_accuracy = [] + for i in loading_bar: + loading_bar.set_description(f"Epoch: {i}") + + state, train_metrics = self._train_epoch( + state, train_ds, batch_size=batch_size + ) + metrics = self._evaluate_model(state.params, test_ds) + + loading_bar.set_postfix(test_loss=metrics["loss"]) + if self.accuracy_fn is not None: + loading_bar.set_postfix(accuracy=metrics["accuracy"]) + test_accuracy.append(metrics["accuracy"]) + train_accuracy.append(train_metrics["accuracy"]) + + test_losses.append(metrics["loss"]) + train_losses.append(train_metrics["loss"]) + + # Update the final model state. + self.model_state = state + + return { + "test_losses": test_losses, + "test_accuracy": test_accuracy, + "train_losses": train_losses, + "train_accuracy": train_accuracy, + } + + def __call__(self, feature_vector: jnp.ndarray): + """ + See parent class for full doc string. + """ + state = self.model_state + + return self.apply_fn(state.params, feature_vector) diff --git a/symsuite/symmetry_group_extraction/group_detection.py b/symsuite/symmetry_group_extraction/group_detection.py index 02c6070..f495a86 100644 --- a/symsuite/symmetry_group_extraction/group_detection.py +++ b/symsuite/symmetry_group_extraction/group_detection.py @@ -10,12 +10,18 @@ =============== Cluster raw data into symmetry groups """ -from symsuite.ml_models.dense_model import DenseModel -from symsuite.analysis.model_visualization import Visualizer from typing import Tuple + +import jax.numpy as jnp import numpy as np -import tensorflow as tf -from symsuite.utils.data_clustering import compute_com, compute_radius_of_gyration + +from symsuite.analysis.model_visualization import Visualizer +from symsuite.ml_models.dense_model import DenseModel +from symsuite.utils.data_clustering import ( + compute_com, + compute_radius_of_gyration, + to_categorical, +) class GroupDetection: @@ -32,57 +38,22 @@ class GroupDetection: Which set to use in the representation, train, validation, or test. """ - def __init__(self, model: DenseModel, data_clusters: dict, representation_set: str = 'train'): + def __init__( + self, model_representation: np.ndarray, data_classes: list + ): """ Constructor for the GroupDetection class. Parameters ---------- - model : DenseModel - Model to use in the group detection. - data_clusters : dict - Data cluster class used for the partitioning of the data. - representation_set : str - Which set to use in the representation, train, validation, or test. - """ - self.model = model - self.data = data_clusters - self.representation_set = representation_set - self.model.add_data(self.data) # add the data to the model. - - def _get_model_predictions(self) -> Tuple: - """ - Train the attached model. - - Returns - ------- - val_data : tf.Tensor - Data on which the prediction were made. - model_predictions : Tuple - Embedding layer of the NN on validation data. - """ - self.model.train_model() - if self.representation_set == 'train': - val_data = self.model.train_ds - predictions = self.model.model.predict(val_data[:, 0:self.model.input_shape]) - elif self.representation_set == 'test:': - val_data = self.model.test_ds - predictions = self.model.model.predict(val_data[:, 0:self.model.input_shape]) - else: - val_data = self.model.val_ds - predictions = self.model.model.predict(val_data[:, 0:self.model.input_shape]) - - return val_data, predictions - - def _run_visualization(self): - """ - Perform a visualization on the TSNE data. - - Returns - ------- - + model_representation : np.ndarray + Model representation on which to perform the symmetry connection + analysis. + data_classes : list + List of the data classes for better visualization """ - pass + self.model_representations = model_representation + self.data_classes = data_classes @staticmethod def _cluster_detection(function_data: np.ndarray, data: np.ndarray): @@ -103,50 +74,23 @@ def _cluster_detection(function_data: np.ndarray, data: np.ndarray): e.g. {1: [radial values], 2: [radial_values], ...} """ net_array = np.concatenate((data, function_data), 1) - sorted_data = tf.gather(net_array, tf.argsort(net_array[:, -1])).numpy() + sorted_data = jnp.take(net_array, jnp.argsort(net_array[:, -1])) class_array = np.unique(function_data[:, -1]) point_cloud = {} # loop over the class array for i, item in enumerate(class_array): - start = np.searchsorted(sorted_data[:, -1], item, side='left') - stop = np.searchsorted(sorted_data[:, -1], item, side='right') - 1 + start = np.searchsorted(sorted_data[:, -1], item, side="left") + stop = np.searchsorted(sorted_data[:, -1], item, side="right") - 1 com = compute_com(sorted_data[start:stop, 0:2]) rg = compute_radius_of_gyration(sorted_data[start:stop, 0:2], com) - #print(f"Class: {item}, COM: {com}, Rg: {rg}") + # print(f"Class: {item}, COM: {com}, Rg: {rg}") if rg > 1000: point_cloud[item] = sorted_data[start:stop, 2:-1] return point_cloud - @staticmethod - def _filter_data(predictions: tf.Tensor, targets: tf.Tensor): - """ - Check which data points are predicted well and include them in the data. - - Parameters - ---------- - targets : tf.Tensor - Target values on which predictions were made. - predictions : tf.Tensor - Network predictions. - - Returns - ------- - - """ - accepted_candidates = np.zeros(len(predictions)) - target_values = tf.keras.utils.to_categorical(targets[:, -1]) - counter = 0 - for i, item in enumerate(predictions): - if np.linalg.norm(predictions[i] - target_values[i]) <= 2e-1: - accepted_candidates[counter] = i - counter += 1 - accepted_candidates = tf.convert_to_tensor(accepted_candidates[0:counter], dtype=tf.int32) - - return tf.gather(targets, accepted_candidates) - def run_symmetry_detection(self, plot: bool = True, save: bool = False): """ Run the symmetry detection routine. @@ -163,7 +107,9 @@ def run_symmetry_detection(self, plot: bool = True, save: bool = False): """ validation_data, predictions = self._get_model_predictions() accepted_data = self._filter_data(predictions, validation_data) - representation = self.model.get_embedding_layer_representation(accepted_data) # get the embedding layer + representation = self.model.get_embedding_layer_representation( + accepted_data + ) # get the embedding layer visualizer = Visualizer(representation, accepted_data[:, -1]) data = visualizer.tsne_visualization(plot=plot, save=save) diff --git a/symsuite/utils/data_clustering.py b/symsuite/utils/data_clustering.py index 84265cf..5dd77a7 100644 --- a/symsuite/utils/data_clustering.py +++ b/symsuite/utils/data_clustering.py @@ -8,13 +8,13 @@ Description: Methods to help with clustering data. """ -import tensorflow as tf -import numpy as np from typing import Tuple -import sys + +import jax.numpy as jnp +import numpy as np -def _build_condlist(data: np.array, bin_values: dict) -> Tuple: +def _build_condition_list(data: np.array, bin_values: dict) -> Tuple: """ Build the condition list for the piecewise implementation. @@ -37,16 +37,14 @@ def _build_condlist(data: np.array, bin_values: dict) -> Tuple: classes = [] for key in bin_values: conditions.append( - np.logical_and( - data >= (bin_values[key][0]), data <= (bin_values[key][1]) - ) + np.logical_and(data >= (bin_values[key][0]), data <= (bin_values[key][1])) ) classes.append(key) return conditions, classes -def _function_to_bins(function_values: tf.Tensor, bin_values: dict) -> tf.Tensor: +def _function_to_bins(function_values: jnp.ndarray, bin_values: dict) -> jnp.ndarray: """ Sort function values into bins. @@ -59,29 +57,49 @@ def _function_to_bins(function_values: tf.Tensor, bin_values: dict) -> tf.Tensor Returns ------- - conditions : tf.Tensor + conditions : jnp.ndarrau Conditions from the cond list build. """ - conditions, functions = _build_condlist(function_values, bin_values) + conditions, functions = _build_condition_list(function_values, bin_values) + + return jnp.array(conditions) + + +def to_categorical(data: jnp.ndarray): + """ + Implementation of the keras.to_categorical function + + Parameters + ---------- + data : jnp.ndarray (n_points,) + + Returns + ------- + categorical_data : jnp.ndarray (n_points, n_classes) + Data converted into categorical format. + """ + order = int(max(data) + 1) + classes = jnp.eye(order) - return tf.convert_to_tensor(conditions) + return jnp.take(classes, data, axis=0) def range_binning( - image: tf.Tensor, - domain: tf.Tensor, - value_range: list, - bin_operation: list, - representatives: int = 100) -> dict: + image: jnp.ndarray, + domain: jnp.ndarray, + value_range: list, + bin_operation: list, + representatives: int = 100, +) -> dict: """ A method to apply simple range binning to some data. Parameters ---------- - image : tf.Tensor + image : jnp.ndarrau data to cluster. - domain : tf.Tensor + domain : jnp.ndarrau data pool to return clustered. representatives : int Number of class representatives to have for each bin. @@ -110,7 +128,7 @@ def range_binning( bin_masks = _function_to_bins(image, bin_values) # Check that there is enough data in each class. - bin_count = tf.reduce_sum(tf.cast(bin_masks, tf.int8), 1) + bin_count = jnp.sum(bin_masks.astype(int), axis=1) if any(bin_count) < representatives: print("WARNING: Not enough data! Some classes will be under-represented.") @@ -119,10 +137,10 @@ def range_binning( clustered_data = {} for i in range(len(class_keys)): clustered_data[class_keys[i]] = {} - filtered_domain = tf.boolean_mask(domain, bin_masks[i]) - filtered_image = tf.boolean_mask(image, bin_masks[i]) - clustered_data[class_keys[i]]['domain'] = filtered_domain[0:representatives] - clustered_data[class_keys[i]]['image'] = filtered_image[0:representatives] + filtered_domain = domain[bin_masks[i]] + filtered_image = image[bin_masks[i]] + clustered_data[class_keys[i]]["domain"] = filtered_domain[0:representatives] + clustered_data[class_keys[i]]["image"] = filtered_image[0:representatives] return clustered_data @@ -140,7 +158,7 @@ def compute_com(data: np.ndarray): ------- """ - return tf.reduce_mean(data, axis=0) + return jnp.mean(data, axis=0) def compute_radius_of_gyration(data: np.ndarray, com: np.ndarray): @@ -156,6 +174,6 @@ def compute_radius_of_gyration(data: np.ndarray, com: np.ndarray): ------- """ - rg_primitive = tf.reduce_sum((data - com)**2, axis=1) + rg_primitive = jnp.sum((data - com) ** 2, axis=1) - return tf.reduce_mean(rg_primitive, axis=0) + return jnp.mean(rg_primitive, axis=0)