From 1af3a40a9c7df399d9417d6e2ca8e573ecb51ed2 Mon Sep 17 00:00:00 2001 From: Dhruv Sondhi <66117751+DhruvSondhi@users.noreply.github.com> Date: Mon, 12 Jul 2021 22:35:39 +0530 Subject: [PATCH] Moved Simulation & Montecarlo Logging Frameworks to `tardis/io/logger` folder (#1684) * Moved Logging Frameworks to tardis/io/logger folder * Fixing some errors with the tests * Changed simulation_logger.py to logger.py * Changed implementation to target only TARDIS loggers Removed list_of_filters to check the filters from the logger itself * Removed list creation in logging_state & Moved in tardis_logger() --- tardis/__init__.py | 143 --------------- tardis/base.py | 2 +- tardis/io/logger/__init__.py | 0 tardis/{util => io/logger}/colored_logger.py | 0 tardis/io/logger/logger.py | 163 ++++++++++++++++++ .../logger}/montecarlo_logger.py | 0 tardis/io/logger/tests/__init__.py | 0 tardis/io/logger/tests/test_logging.py | 99 +++++++++++ tardis/montecarlo/base.py | 2 +- .../montecarlo_numba/single_packet_loop.py | 4 +- tardis/simulation/tests/test_simulation.py | 95 +--------- 11 files changed, 267 insertions(+), 241 deletions(-) create mode 100644 tardis/io/logger/__init__.py rename tardis/{util => io/logger}/colored_logger.py (100%) create mode 100644 tardis/io/logger/logger.py rename tardis/{montecarlo/montecarlo_numba => io/logger}/montecarlo_logger.py (100%) create mode 100644 tardis/io/logger/tests/__init__.py create mode 100644 tardis/io/logger/tests/test_logging.py diff --git a/tardis/__init__.py b/tardis/__init__.py index 0f83c3bb2c3..820cffad225 100644 --- a/tardis/__init__.py +++ b/tardis/__init__.py @@ -12,156 +12,13 @@ # ---------------------------------------------------------------------------- import sys -import logging -import warnings import pyne.data -from tardis.util.colored_logger import ColoredFormatter, formatter_message # ---------------------------------------------------------------------------- from tardis.base import run_tardis from tardis.io.util import yaml_load_config_file as yaml_load -warnings.filterwarnings("ignore", category=pyne.utils.QAWarning) - -FORMAT = "[$BOLD%(name)-20s$RESET][%(levelname)-18s] %(message)s ($BOLD%(filename)s$RESET:%(lineno)d)" -COLOR_FORMAT = formatter_message(FORMAT, True) - -logging.captureWarnings(True) -logger = logging.getLogger("tardis") -logger.setLevel(logging.INFO) - -console_handler = logging.StreamHandler(sys.stdout) -console_formatter = ColoredFormatter(COLOR_FORMAT) -console_handler.setFormatter(console_formatter) - -logger.addHandler(console_handler) -logging.getLogger("py.warnings").addHandler(console_handler) - -LOGGING_LEVELS = { - "NOTSET": logging.NOTSET, - "DEBUG": logging.DEBUG, - "INFO": logging.INFO, - "WARNING": logging.WARNING, - "ERROR": logging.ERROR, - "CRITICAL": logging.CRITICAL, -} -DEFAULT_LOG_STATE = "CRITICAL" - - -class FilterLog(object): - """ - Filter Log Class for Filtering Logging Output - to a particular level - - Parameters - ---------- - log_level : logging object - allows to have a filter for the - particular log_level - """ - - def __init__(self, log_level): - self.log_level = log_level - - def filter(self, log_record): - """ - filter() allows to set the logging level for - all the record that are being parsed & hence remove those - which are not of the particular level - - Parameters - ---------- - log_record : logging.record - which the paricular record upon which the - filter will be applied - - Returns - ------- - boolean : True, if the current log_record has the - level that of the specified log_level - False, if the current log_record doesn't have the - same log_level as the specified one - """ - return log_record.levelno == self.log_level - - -# Setting up a list to store the Logging Filters set by logger.addFilter() -list_of_filter = [] - - -def logging_state(log_state, tardis_config, specific): - """ - Function to set the logging configuration for the simulation output - Called from within run_tardis() - Configured via functional arguments passed through run_tardis() - log_state & specific - Configured via YAML parameters under `debug` section - logging_level & specific_logging - - Parameters - ---------- - log_state: str - Allows to input the log level for the simulation - Uses Python logging framework to determine the messages that will be output - specific: boolean - Allows to set specific logging levels. Logs of the `log_state` level would be output. - """ - - if "debug" in tardis_config: - specific = ( - tardis_config["debug"]["specific"] if specific is None else specific - ) - - logging_level = ( - log_state if log_state else tardis_config["debug"]["log_state"] - ) - - # Displays a message when both log_state & tardis["debug"]["log_state"] are specified - if log_state and tardis_config["debug"]["log_state"]: - print( - "log_state is defined both in Functional Argument & YAML Configuration {debug section}" - ) - print( - f"log_state = {log_state.upper()} will be used for Log Level Determination\n" - ) - - else: - if log_state: - logging_level = log_state - else: - tardis_config["debug"] = {"log_state": DEFAULT_LOG_STATE} - logging_level = tardis_config["debug"]["log_state"] - - if specific: - specific = specific - - logging_level = logging_level.upper() - if not logging_level in LOGGING_LEVELS: - raise ValueError( - f"Passed Value for log_state = {logging_level} is Invalid. Must be one of the following {list(LOGGING_LEVELS.keys())}" - ) - - loggers = [ - logging.getLogger(name) for name in logging.root.manager.loggerDict - ] - if logging_level in LOGGING_LEVELS: - for logger in loggers: - logger.setLevel(LOGGING_LEVELS[logging_level]) - - if list_of_filter: - for filter in list_of_filter: - for logger in loggers: - logger.removeFilter(filter) - - if specific: - filter_log = FilterLog(LOGGING_LEVELS[logging_level]) - list_of_filter.append(filter_log) - for logger in loggers: - logger.addFilter(filter_log) - else: - for filter in list_of_filter: - for logger in loggers: - logger.removeFilter(filter) - # ---------------------------------------------------------------------------- # pyne holds Python 3.7 on macOS, but refdata is pickled with protocol 5 (3.8.3) diff --git a/tardis/base.py b/tardis/base.py index e9e03fe42bb..2cc3e0c2444 100644 --- a/tardis/base.py +++ b/tardis/base.py @@ -33,7 +33,7 @@ def run_tardis( ------- Simulation """ - from tardis import logging_state + from tardis.io.logger.logger import logging_state from tardis.io.config_reader import Configuration from tardis.io.atom_data.base import AtomData from tardis.simulation import Simulation diff --git a/tardis/io/logger/__init__.py b/tardis/io/logger/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tardis/util/colored_logger.py b/tardis/io/logger/colored_logger.py similarity index 100% rename from tardis/util/colored_logger.py rename to tardis/io/logger/colored_logger.py diff --git a/tardis/io/logger/logger.py b/tardis/io/logger/logger.py new file mode 100644 index 00000000000..4ae8369d548 --- /dev/null +++ b/tardis/io/logger/logger.py @@ -0,0 +1,163 @@ +import logging +import sys +import logging +import warnings +import pyne.data +from tardis.io.logger.colored_logger import ColoredFormatter, formatter_message + +warnings.filterwarnings("ignore", category=pyne.utils.QAWarning) + +FORMAT = "[$BOLD%(name)-20s$RESET][%(levelname)-18s] %(message)s ($BOLD%(filename)s$RESET:%(lineno)d)" +COLOR_FORMAT = formatter_message(FORMAT, True) + +logging.captureWarnings(True) +logger = logging.getLogger("tardis") +logger.setLevel(logging.INFO) + +console_handler = logging.StreamHandler(sys.stdout) +console_formatter = ColoredFormatter(COLOR_FORMAT) +console_handler.setFormatter(console_formatter) + +logger.addHandler(console_handler) +logging.getLogger("py.warnings").addHandler(console_handler) + +LOGGING_LEVELS = { + "NOTSET": logging.NOTSET, + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "CRITICAL": logging.CRITICAL, +} +DEFAULT_LOG_STATE = "CRITICAL" + + +class FilterLog(object): + """ + Filter Log Class for Filtering Logging Output + to a particular level + + Parameters + ---------- + log_level : logging object + allows to have a filter for the + particular log_level + """ + + def __init__(self, log_level): + self.log_level = log_level + + def filter(self, log_record): + """ + filter() allows to set the logging level for + all the record that are being parsed & hence remove those + which are not of the particular level + + Parameters + ---------- + log_record : logging.record + which the paricular record upon which the + filter will be applied + + Returns + ------- + boolean : True, if the current log_record has the + level that of the specified log_level + False, if the current log_record doesn't have the + same log_level as the specified one + """ + return log_record.levelno == self.log_level + + +def logging_state(log_state, tardis_config, specific): + """ + Function to set the logging configuration for the simulation output + Called from within run_tardis() + Configured via functional arguments passed through run_tardis() - log_state & specific + Configured via YAML parameters under `debug` section - logging_level & specific_logging + Parameters + ---------- + log_state: str + Allows to input the log level for the simulation + Uses Python logging framework to determine the messages that will be output + specific: boolean + Allows to set specific logging levels. Logs of the `log_state` level would be output. + """ + + if "debug" in tardis_config: + specific = ( + tardis_config["debug"]["specific"] if specific is None else specific + ) + + logging_level = ( + log_state if log_state else tardis_config["debug"]["log_state"] + ) + + # Displays a message when both log_state & tardis["debug"]["log_state"] are specified + if log_state and tardis_config["debug"]["log_state"]: + print( + "log_state is defined both in Functional Argument & YAML Configuration {debug section}" + ) + print( + f"log_state = {log_state.upper()} will be used for Log Level Determination\n" + ) + + else: + if log_state: + logging_level = log_state + else: + tardis_config["debug"] = {"log_state": DEFAULT_LOG_STATE} + logging_level = tardis_config["debug"]["log_state"] + + if specific: + specific = specific + + logging_level = logging_level.upper() + if not logging_level in LOGGING_LEVELS: + raise ValueError( + f"Passed Value for log_state = {logging_level} is Invalid. Must be one of the following {list(LOGGING_LEVELS.keys())}" + ) + + # Getting the TARDIS logger & all its children loggers + logger = logging.getLogger("tardis") + + # Creating a list for Storing all the Loggers which are derived from TARDIS + tardis_loggers = tardis_logger() + + if logging_level in LOGGING_LEVELS: + logger.setLevel(LOGGING_LEVELS[logging_level]) + + if logger.filters: + for filter in logger.filters: + for logger in tardis_loggers: + logger.removeFilter(filter) + + if specific: + filter_log = FilterLog(LOGGING_LEVELS[logging_level]) + for logger in tardis_loggers: + logger.addFilter(filter_log) + else: + for filter in logger.filters: + for logger in tardis_loggers: + logger.removeFilter(filter) + + +def tardis_logger(): + """ + Generates the list of the loggers which are derived from TARDIS + All loggers which are of the form `tardis.module_name` are added to the list + + Parameters + ---------- + list_for_loggers : list + List for storing the loggers derived from TARDIS + + Returns + ------- + list_for_loggers : list + """ + list_for_loggers = [] + for name in logging.root.manager.loggerDict: + if not name.find("tardis"): + list_for_loggers.append(logging.getLogger(name)) + return list_for_loggers diff --git a/tardis/montecarlo/montecarlo_numba/montecarlo_logger.py b/tardis/io/logger/montecarlo_logger.py similarity index 100% rename from tardis/montecarlo/montecarlo_numba/montecarlo_logger.py rename to tardis/io/logger/montecarlo_logger.py diff --git a/tardis/io/logger/tests/__init__.py b/tardis/io/logger/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tardis/io/logger/tests/test_logging.py b/tardis/io/logger/tests/test_logging.py new file mode 100644 index 00000000000..4d7d370bb0c --- /dev/null +++ b/tardis/io/logger/tests/test_logging.py @@ -0,0 +1,99 @@ +import pytest +import logging + +from tardis.io.config_reader import Configuration +from tardis.simulation import Simulation +from tardis.io.logger.logger import LOGGING_LEVELS +from tardis import run_tardis + + +def test_logging_simulation(atomic_data_fname, caplog): + """ + Testing the logs for simulations runs + """ + config = Configuration.from_yaml( + "tardis/io/tests/data/tardis_configv1_verysimple.yml" + ) + config["atom_data"] = atomic_data_fname + + simulation = Simulation.from_config(config) + + simulation.run() + + for record in caplog.records: + assert record.levelno >= logging.INFO + + +# Testing Configuration of Logger via run_tardis() Function +@pytest.mark.parametrize( + ["log_state", "specific"], + [ + ("Info", False), + ("INFO", False), + ("INFO", True), + ("DEBUG", False), + ("DEBUG", True), + ("WARNING", True), + ("ERROR", True), + ("CRITICAL", True), + ("NOTSET", False), + ], +) +class TestSimulationLogging: + """ + Class implemented for testing the logging configuration available via run_tardis() + Tests Functional Arguments : log_state & specific + Tests YAML Parameters : logging_level & specific_logging + """ + + def test_logging_config( + self, atomic_data_fname, caplog, log_state, specific + ): + config = Configuration.from_yaml( + "tardis/io/tests/data/tardis_configv1_verysimple_logger.yml" + ) + config["atom_data"] = atomic_data_fname + + caplog.clear() + run_tardis(config=config, log_state=log_state, specific=specific) + for record in caplog.records: + if specific == True: + assert record.levelno == LOGGING_LEVELS[log_state.upper()] + else: + assert record.levelno >= LOGGING_LEVELS[log_state.upper()] + + def test_logging_config_yaml( + self, atomic_data_fname, caplog, log_state, specific + ): + config = Configuration.from_yaml( + "tardis/io/tests/data/tardis_configv1_verysimple_logger.yml" + ) + config["atom_data"] = atomic_data_fname + config["debug"]["log_state"] = log_state + config["debug"]["specific"] = specific + + caplog.clear() + run_tardis(config=config) + for record in caplog.records: + if specific == True: + assert record.levelno == LOGGING_LEVELS[log_state.upper()] + else: + assert record.levelno >= LOGGING_LEVELS[log_state.upper()] + + def test_logging_both_specified( + self, atomic_data_fname, caplog, log_state, specific + ): + config = Configuration.from_yaml( + "tardis/io/tests/data/tardis_configv1_verysimple_logger.yml" + ) + config["atom_data"] = atomic_data_fname + config["debug"]["log_state"] = log_state + config["debug"]["specific"] = specific + + caplog.clear() + run_tardis(config=config, log_state=log_state, specific=specific) + for record in caplog.records: + if specific == True: + assert record.levelno == LOGGING_LEVELS[log_state.upper()] + else: + assert record.levelno >= LOGGING_LEVELS[log_state.upper()] diff --git a/tardis/montecarlo/base.py b/tardis/montecarlo/base.py index a78d82c50b4..d1fa24e1f30 100644 --- a/tardis/montecarlo/base.py +++ b/tardis/montecarlo/base.py @@ -17,11 +17,11 @@ from tardis.montecarlo.montecarlo_numba import montecarlo_radial1d -from tardis.montecarlo.montecarlo_numba import montecarlo_logger as mc_logger from tardis.montecarlo.montecarlo_numba.numba_interface import ( configuration_initialize, ) from tardis.montecarlo.montecarlo_numba import numba_config +from tardis.io.logger import montecarlo_logger as mc_logger import numpy as np diff --git a/tardis/montecarlo/montecarlo_numba/single_packet_loop.py b/tardis/montecarlo/montecarlo_numba/single_packet_loop.py index 525e4402af6..c14321c8079 100644 --- a/tardis/montecarlo/montecarlo_numba/single_packet_loop.py +++ b/tardis/montecarlo/montecarlo_numba/single_packet_loop.py @@ -31,8 +31,8 @@ C_SPEED_OF_LIGHT = const.c.to("cm/s").value -from tardis.montecarlo.montecarlo_numba.montecarlo_logger import log_decorator -from tardis.montecarlo.montecarlo_numba import montecarlo_logger as mc_logger +from tardis.io.logger.montecarlo_logger import log_decorator +from tardis.io.logger import montecarlo_logger as mc_logger # @log_decorator @njit diff --git a/tardis/simulation/tests/test_simulation.py b/tardis/simulation/tests/test_simulation.py index ea6b0f55ffe..88f3daff827 100644 --- a/tardis/simulation/tests/test_simulation.py +++ b/tardis/simulation/tests/test_simulation.py @@ -5,7 +5,7 @@ from tardis.io.config_reader import Configuration from tardis.simulation import Simulation -from tardis import run_tardis, LOGGING_LEVELS +from tardis import run_tardis import numpy as np import pandas as pd @@ -156,96 +156,3 @@ def test_plasma_state_storer_reshape( # assert_quantity_allclose( # t_rad, simulation_compare_data['test1/t_rad'] * u.Unit('K'), atol=0.0 * u.Unit('K')) - - -def test_logging_simulation(atomic_data_fname, caplog): - """ - Testing the logs for simulations runs - """ - config = Configuration.from_yaml( - "tardis/io/tests/data/tardis_configv1_verysimple.yml" - ) - config["atom_data"] = atomic_data_fname - - simulation = Simulation.from_config(config) - - simulation.run() - - for record in caplog.records: - assert record.levelno >= logging.INFO - - -# Testing Configuration of Logger via run_tardis() Function -# TODO: Move to its own Testing Module (Logger) -@pytest.mark.parametrize( - ["log_state", "specific"], - [ - ("Info", False), - ("INFO", False), - ("INFO", True), - ("DEBUG", False), - ("DEBUG", True), - ("WARNING", True), - ("ERROR", True), - ("CRITICAL", True), - ("NOTSET", False), - ], -) -class TestSimulationLogging: - """ - Class implemented for testing the logging configuration available via run_tardis() - Tests Functional Arguments : log_state & specific - Tests YAML Parameters : logging_level & specific_logging - """ - - def test_logging_config( - self, atomic_data_fname, caplog, log_state, specific - ): - config = Configuration.from_yaml( - "tardis/io/tests/data/tardis_configv1_verysimple_logger.yml" - ) - config["atom_data"] = atomic_data_fname - - caplog.clear() - run_tardis(config=config, log_state=log_state, specific=specific) - for record in caplog.records: - if specific == True: - assert record.levelno == LOGGING_LEVELS[log_state.upper()] - else: - assert record.levelno >= LOGGING_LEVELS[log_state.upper()] - - def test_logging_config_yaml( - self, atomic_data_fname, caplog, log_state, specific - ): - config = Configuration.from_yaml( - "tardis/io/tests/data/tardis_configv1_verysimple_logger.yml" - ) - config["atom_data"] = atomic_data_fname - config["debug"]["log_state"] = log_state - config["debug"]["specific"] = specific - - caplog.clear() - run_tardis(config=config) - for record in caplog.records: - if specific == True: - assert record.levelno == LOGGING_LEVELS[log_state.upper()] - else: - assert record.levelno >= LOGGING_LEVELS[log_state.upper()] - - def test_logging_both_specified( - self, atomic_data_fname, caplog, log_state, specific - ): - config = Configuration.from_yaml( - "tardis/io/tests/data/tardis_configv1_verysimple_logger.yml" - ) - config["atom_data"] = atomic_data_fname - config["debug"]["log_state"] = log_state - config["debug"]["specific"] = specific - - caplog.clear() - run_tardis(config=config, log_state=log_state, specific=specific) - for record in caplog.records: - if specific == True: - assert record.levelno == LOGGING_LEVELS[log_state.upper()] - else: - assert record.levelno >= LOGGING_LEVELS[log_state.upper()]