From e91a7b70a62fe660bd6a616240e2b77e3425108a Mon Sep 17 00:00:00 2001 From: Wolfgang Kerzendorf Date: Thu, 2 Aug 2018 14:49:25 +0200 Subject: [PATCH] Refactor logging (#857) * remove wrong comment from atomic. use stdout for logging * make util a package fix double logging problem --- .../plasma_plots/lte_ionization_balance.py | 4 +- .../nebular_ionization_balance.py | 6 +- tardis/__init__.py | 14 ++-- tardis/gui/widgets.py | 5 +- tardis/io/atomic.py | 2 +- tardis/io/model_reader.py | 3 +- tardis/io/util.py | 68 +------------------ tardis/model/base.py | 2 +- tardis/model/density.py | 2 +- tardis/montecarlo/base.py | 2 +- .../montecarlo/tests/test_formal_integral.py | 2 +- tardis/plasma/properties/j_blues.py | 2 +- tardis/plasma/standard_plasmas.py | 2 +- tardis/tests/test_util.py | 10 ++- tardis/util/__init__.py | 3 + tardis/{util.py => util/base.py} | 27 ++++---- tardis/util/colored_logger.py | 66 ++++++++++++++++++ 17 files changed, 119 insertions(+), 101 deletions(-) create mode 100644 tardis/util/__init__.py rename tardis/{util.py => util/base.py} (99%) create mode 100644 tardis/util/colored_logger.py diff --git a/docs/physics/plasma/plasma_plots/lte_ionization_balance.py b/docs/physics/plasma/plasma_plots/lte_ionization_balance.py index 3b510008798..7a8e1951bce 100644 --- a/docs/physics/plasma/plasma_plots/lte_ionization_balance.py +++ b/docs/physics/plasma/plasma_plots/lte_ionization_balance.py @@ -1,6 +1,8 @@ import os from matplotlib import pyplot as plt from matplotlib import colors + +import tardis.util.base from tardis import util from tardis.io import atomic import numpy as np @@ -63,7 +65,7 @@ for ion_number in [0, 1, 2, 3]: current_ion_density = ion_number_densities.ix[14, ion_number] ax1.plot(current_ion_density.index, current_ion_density.values, '%s-' % ion_colors[ion_number], - label='Si %s W=1.0' % util.int_to_roman(ion_number + 1).upper()) + label='Si %s W=1.0' % tardis.util.base.int_to_roman(ion_number + 1).upper()) #only plotting every 5th radiation temperature diff --git a/docs/physics/plasma/plasma_plots/nebular_ionization_balance.py b/docs/physics/plasma/plasma_plots/nebular_ionization_balance.py index 4b22cea808c..428f9125e24 100644 --- a/docs/physics/plasma/plasma_plots/nebular_ionization_balance.py +++ b/docs/physics/plasma/plasma_plots/nebular_ionization_balance.py @@ -1,6 +1,8 @@ import os from matplotlib import colors + +import tardis.util.base from tardis import util from tardis.io import atomic from matplotlib import pyplot as plt @@ -58,7 +60,7 @@ for ion_number in [0, 1, 2, 3]: current_ion_density = ion_number_densities.ix[14, ion_number] ax1.plot(current_ion_density.index, current_ion_density.values, '%s-' % ion_colors[ion_number], - label='Si %s W=1.0' % util.int_to_roman(ion_number + 1).upper()) + label='Si %s W=1.0' % tardis.util.base.int_to_roman(ion_number + 1).upper()) #only plotting every 5th radiation temperature @@ -92,7 +94,7 @@ print "w=0.5" current_ion_density = ion_number_densities.ix[14, ion_number] ax1.plot(current_ion_density.index, current_ion_density.values, '%s--' % ion_colors[ion_number], - label='Si %s W=0.5' % util.int_to_roman(ion_number + 1).upper()) + label='Si %s W=0.5' % tardis.util.base.int_to_roman(ion_number + 1).upper()) for t_rad in t_rads[::5]: ax2.plot(level_populations[t_rad].index, level_populations[t_rad].values, color=t_rad_color_map.to_rgba(t_rad), diff --git a/tardis/__init__.py b/tardis/__init__.py index 5027acb3fb7..9e2722557f2 100644 --- a/tardis/__init__.py +++ b/tardis/__init__.py @@ -1,6 +1,6 @@ # Licensed under a 3-clause BSD style license - see LICENSE.rst -import logging -from tardis.io.util import ColoredLogger +import logging, sys +from tardis.util.colored_logger import ColoredFormatter, formatter_message # Affiliated packages may add whatever they like to this file, but # should keep this content at the top. # ---------------------------------------------------------------------------- @@ -10,12 +10,16 @@ from tardis.base import run_tardis from tardis.io.util import yaml_load_config_file as yaml_load +FORMAT = "[$BOLD%(name)-20s$RESET][%(levelname)-18s] %(message)s ($BOLD%(filename)s$RESET:%(lineno)d)" +COLOR_FORMAT = formatter_message(FORMAT, True) + logging.captureWarnings(True) logger = logging.getLogger('tardis') -logging.setLoggerClass(ColoredLogger) logger.setLevel(logging.INFO) -console_handler = logging.StreamHandler() -console_formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s') + +console_handler = logging.StreamHandler(sys.stdout) +console_formatter = ColoredFormatter(COLOR_FORMAT) console_handler.setFormatter(console_formatter) + logger.addHandler(console_handler) logging.getLogger('py.warnings').addHandler(console_handler) diff --git a/tardis/gui/widgets.py b/tardis/gui/widgets.py index e819af68c6a..8c90b4014c5 100644 --- a/tardis/gui/widgets.py +++ b/tardis/gui/widgets.py @@ -1,4 +1,7 @@ import os + +import tardis.util.base + if os.environ.get('QT_API', None)=='pyqt': from PyQt4 import QtGui, QtCore elif os.environ.get('QT_API', None)=='pyside': @@ -969,7 +972,7 @@ def __init__(self, line_interaction_analysis, atom_data, lines_data, description 'ion_number']) self.species_selected = sorted( line_interaction_species_group.groups.keys()) - species_symbols = [util.species_tuple_to_string(item) for item in self.species_selected] + species_symbols = [tardis.util.base.species_tuple_to_string(item) for item in self.species_selected] species_table_model = self.createTable([species_symbols, ['Species']]) species_abundances = ( line_interaction_species_group.wavelength.count().astype(float) / diff --git a/tardis/io/atomic.py b/tardis/io/atomic.py index 7af9fc3423c..40d0fb3cef3 100644 --- a/tardis/io/atomic.py +++ b/tardis/io/atomic.py @@ -375,7 +375,7 @@ def prepare_atom_data( self.macro_atom_data.loc[:, 'destination_level_idx'] = self.macro_atom_references.loc[ tmp_macro_destination_level_idx, "references_idx" - ].astype(np.int64).values # why it is named `destination_level_idx` ?! It is reference index + ].astype(np.int64).values self.macro_atom_data.loc[:, 'source_level_idx'] = self.macro_atom_references.loc[ tmp_macro_source_level_idx, "references_idx" diff --git a/tardis/io/model_reader.py b/tardis/io/model_reader.py index b18d27535a9..237df902e20 100644 --- a/tardis/io/model_reader.py +++ b/tardis/io/model_reader.py @@ -11,7 +11,8 @@ # Adding logging support logger = logging.getLogger(__name__) -from tardis.util import parse_quantity +from tardis.util.base import parse_quantity + class ConfigurationError(Exception): pass diff --git a/tardis/io/util.py b/tardis/io/util.py index 6bdc3d5a388..bbd0270490b 100644 --- a/tardis/io/util.py +++ b/tardis/io/util.py @@ -8,7 +8,7 @@ from collections import OrderedDict import yaml from astropy import constants, units as u -from tardis.util import element_symbol2atomic_number +from tardis.util.base import element_symbol2atomic_number import logging logger = logging.getLogger(__name__) @@ -338,67 +338,5 @@ def to_hdf(self, file_path, path='', name=None, collection=None): super(PlasmaWriterMixin, self).to_hdf(file_path, path, name) -''' -Code for Custom Logger Classes (ColoredFormatter and ColorLogger) and its helper function -(formatter_message) is used from this thread -http://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output -''' -def formatter_message(message, use_color=True): - ''' - Helper Function used for Coloring Log Output - ''' - #These are the sequences need to get colored ouput - RESET_SEQ = "\033[0m" - BOLD_SEQ = "\033[1m" - if use_color: - message = message.replace( - "$RESET", RESET_SEQ).replace("$BOLD", BOLD_SEQ) - else: - message = message.replace("$RESET", "").replace("$BOLD", "") - return message - - -class ColoredFormatter(logging.Formatter): - ''' - Custom logger class for changing levels color - ''' - def __init__(self, msg, use_color=True): - logging.Formatter.__init__(self, msg) - self.use_color = use_color - - def format(self, record): - COLOR_SEQ = "\033[1;%dm" - RESET_SEQ = "\033[0m" - BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) - COLORS = { - 'WARNING': YELLOW, - 'INFO': WHITE, - 'DEBUG': BLUE, - 'CRITICAL': YELLOW, - 'ERROR': RED - } - levelname = record.levelname - if self.use_color and levelname in COLORS: - levelname_color = COLOR_SEQ % ( - 30 + COLORS[levelname]) + levelname + RESET_SEQ - record.levelname = levelname_color - return logging.Formatter.format(self, record) - - -class ColoredLogger(logging.Logger): - ''' - Custom logger class with multiple destinations - ''' - FORMAT = "[$BOLD%(name)-20s$RESET][%(levelname)-18s] %(message)s ($BOLD%(filename)s$RESET:%(lineno)d)" - COLOR_FORMAT = formatter_message(FORMAT, True) - - def __init__(self, name): - logging.Logger.__init__(self, name, logging.DEBUG) - - color_formatter = ColoredFormatter(self.COLOR_FORMAT) - - console = logging.StreamHandler() - console.setFormatter(color_formatter) - - self.addHandler(console) - return + + diff --git a/tardis/model/base.py b/tardis/model/base.py index 43a36058e8f..8aa25963a03 100644 --- a/tardis/model/base.py +++ b/tardis/model/base.py @@ -4,7 +4,7 @@ import pandas as pd from astropy import constants, units as u -from tardis.util import quantity_linspace +from tardis.util.base import quantity_linspace from tardis.io.model_reader import read_density_file, read_abundances_file, read_uniform_abundances from tardis.io.util import HDFWriterMixin from tardis.io.decay import IsotopeAbundances diff --git a/tardis/model/density.py b/tardis/model/density.py index a7676a6d5c6..5031fa696b1 100644 --- a/tardis/model/density.py +++ b/tardis/model/density.py @@ -1,6 +1,6 @@ import numpy as np -from tardis.util import quantity_linspace +from tardis.util.base import quantity_linspace from tardis.io.util import HDFWriterMixin class HomologousDensity(HDFWriterMixin): diff --git a/tardis/montecarlo/base.py b/tardis/montecarlo/base.py index a12bf64bff2..74989516b66 100644 --- a/tardis/montecarlo/base.py +++ b/tardis/montecarlo/base.py @@ -7,7 +7,7 @@ from scipy.special import zeta from spectrum import TARDISSpectrum -from tardis.util import quantity_linspace +from tardis.util.base import quantity_linspace from tardis.io.util import HDFWriterMixin from tardis.montecarlo import montecarlo, packet_source from tardis.montecarlo.formal_integral import FormalIntegrator diff --git a/tardis/montecarlo/tests/test_formal_integral.py b/tardis/montecarlo/tests/test_formal_integral.py index dbd8385124d..56f7fb02ce3 100644 --- a/tardis/montecarlo/tests/test_formal_integral.py +++ b/tardis/montecarlo/tests/test_formal_integral.py @@ -17,7 +17,7 @@ import numpy.testing as ntest -from tardis.util import intensity_black_body +from tardis.util.base import intensity_black_body from tardis.montecarlo.struct import StorageModel diff --git a/tardis/plasma/properties/j_blues.py b/tardis/plasma/properties/j_blues.py index c0cea96e23b..bdb31b24f17 100644 --- a/tardis/plasma/properties/j_blues.py +++ b/tardis/plasma/properties/j_blues.py @@ -4,7 +4,7 @@ from tardis.plasma.properties.base import ProcessingPlasmaProperty, \ DataFrameInput -from tardis.util import intensity_black_body +from tardis.util.base import intensity_black_body class JBluesBlackBody(ProcessingPlasmaProperty): diff --git a/tardis/plasma/standard_plasmas.py b/tardis/plasma/standard_plasmas.py index dcaa115bda9..33edeed11d3 100644 --- a/tardis/plasma/standard_plasmas.py +++ b/tardis/plasma/standard_plasmas.py @@ -5,7 +5,7 @@ import numpy as np from tardis.io import atomic -from tardis.util import species_string_to_tuple +from tardis.util.base import species_string_to_tuple from tardis.plasma import BasePlasma from tardis.plasma.properties.property_collections import (basic_inputs, basic_properties, lte_excitation_properties, lte_ionization_properties, diff --git a/tardis/tests/test_util.py b/tardis/tests/test_util.py index a027a4ffdd0..95bbfe3b92a 100644 --- a/tardis/tests/test_util.py +++ b/tardis/tests/test_util.py @@ -4,12 +4,10 @@ from astropy import units as u from io import StringIO -from tardis.util import MalformedSpeciesError, MalformedElementSymbolError, MalformedQuantityError -from tardis.util import (int_to_roman, roman_to_int, create_synpp_yaml, - calculate_luminosity, intensity_black_body, savitzky_golay, - species_tuple_to_string, species_string_to_tuple, parse_quantity, - element_symbol2atomic_number, atomic_number2element_symbol, - reformat_element_symbol, quantity_linspace, convert_abundances_format) +from tardis.util.base import MalformedSpeciesError, MalformedElementSymbolError, MalformedQuantityError, int_to_roman, \ + roman_to_int, calculate_luminosity, create_synpp_yaml, intensity_black_body, savitzky_golay, \ + species_tuple_to_string, species_string_to_tuple, parse_quantity, element_symbol2atomic_number, \ + atomic_number2element_symbol, reformat_element_symbol, quantity_linspace, convert_abundances_format data_path = os.path.join('tardis', 'io', 'tests', 'data') diff --git a/tardis/util/__init__.py b/tardis/util/__init__.py new file mode 100644 index 00000000000..0d1b60fa94c --- /dev/null +++ b/tardis/util/__init__.py @@ -0,0 +1,3 @@ +# Utilities for TARDIS + + diff --git a/tardis/util.py b/tardis/util/base.py similarity index 99% rename from tardis/util.py rename to tardis/util/base.py index 6dd260bb0ef..465376f209f 100644 --- a/tardis/util.py +++ b/tardis/util/base.py @@ -1,18 +1,16 @@ -# Utilities for TARDIS +import logging +import os +import re +from collections import OrderedDict -from astropy import units as u, constants -from pyne import nucname import numexpr as ne import numpy as np import pandas as pd -import os import yaml -import re -import tardis -import logging - -from collections import OrderedDict +from astropy import constants, units as u +from pyne import nucname +import tardis k_B_cgs = constants.k_B.cgs.value c_cgs = constants.c.cgs.value @@ -20,9 +18,11 @@ m_e_cgs = constants.m_e.cgs.value e_charge_gauss = constants.e.gauss.value + class MalformedError(Exception): pass + class MalformedSpeciesError(MalformedError): def __init__(self, malformed_element_symbol): @@ -51,7 +51,6 @@ def __str__(self): logger = logging.getLogger(__name__) - tardis_dir = os.path.realpath(tardis.__path__[0]) @@ -68,8 +67,6 @@ def get_tests_data_path(fname): symbol2atomic_number = OrderedDict(zip(atomic_symbols_data['symbol'], atomic_symbols_data['atomic_number'])) atomic_number2symbol = OrderedDict(atomic_symbols_data) - - synpp_default_yaml_fname = get_data_path('synpp_default.yaml') @@ -195,6 +192,7 @@ def calculate_luminosity(spec_fname, distance, wavelength_column=0, wavelength_u return luminosity.value, wavelength.min(), wavelength.max() + def create_synpp_yaml(radial1d_mdl, fname, shell_no=0, lines_db=None): logger.warning('Currently only works with Si and a special setup') if radial1d_mdl.atom_data.synpp_refs is not None: @@ -266,6 +264,7 @@ def intensity_black_body(nu, T): '(exp(h_cgs * nu * beta_rad) -1 )') return intensity + def savitzky_golay(y, window_size, order, deriv=0, rate=1): r"""Smooth (and optionally differentiate) data with a Savitzky-Golay filter. The Savitzky-Golay filter removes high frequency noise from data. @@ -405,12 +404,14 @@ def element_symbol2atomic_number(element_string): raise MalformedElementSymbolError(element_string) return symbol2atomic_number[reformatted_element_string] + def atomic_number2element_symbol(atomic_number): """ Convert atomic number to string symbol """ return atomic_number2symbol[atomic_number] + def reformat_element_symbol(element_string): """ Reformat the string so the first letter is uppercase and all subsequent letters lowercase @@ -458,4 +459,4 @@ def convert_abundances_format(fname, delimiter='\s+'): #Assign header row df.columns = [nucname.name(i) for i in range(1, df.shape[1] + 1)] - return df + return df \ No newline at end of file diff --git a/tardis/util/colored_logger.py b/tardis/util/colored_logger.py new file mode 100644 index 00000000000..37130295345 --- /dev/null +++ b/tardis/util/colored_logger.py @@ -0,0 +1,66 @@ +import logging +''' +Code for Custom Logger Classes (ColoredFormatter and ColorLogger) and its helper function +(formatter_message) is used from this thread +http://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output +''' + +def formatter_message(message, use_color=True): + ''' + Helper Function used for Coloring Log Output + ''' + #These are the sequences need to get colored ouput + RESET_SEQ = "\033[0m" + BOLD_SEQ = "\033[1m" + if use_color: + message = message.replace( + "$RESET", RESET_SEQ).replace("$BOLD", BOLD_SEQ) + else: + message = message.replace("$RESET", "").replace("$BOLD", "") + return message + + +class ColoredFormatter(logging.Formatter): + ''' + Custom logger class for changing levels color + ''' + def __init__(self, msg, use_color=True): + logging.Formatter.__init__(self, msg) + self.use_color = use_color + + def format(self, record): + COLOR_SEQ = "\033[1;%dm" + RESET_SEQ = "\033[0m" + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + COLORS = { + 'WARNING': YELLOW, + 'INFO': WHITE, + 'DEBUG': BLUE, + 'CRITICAL': YELLOW, + 'ERROR': RED + } + levelname = record.levelname + if self.use_color and levelname in COLORS: + levelname_color = COLOR_SEQ % ( + 30 + COLORS[levelname]) + levelname + RESET_SEQ + record.levelname = levelname_color + return logging.Formatter.format(self, record) + + +class ColoredLogger(logging.Logger): + ''' + Custom logger class with multiple destinations + ''' + FORMAT = "[$BOLD%(name)-20s$RESET][%(levelname)-18s] %(message)s ($BOLD%(filename)s$RESET:%(lineno)d)" + COLOR_FORMAT = formatter_message(FORMAT, True) + + def __init__(self, name): + logging.Logger.__init__(self, name, logging.DEBUG) + + color_formatter = ColoredFormatter(self.COLOR_FORMAT) + + console = logging.StreamHandler() + console.setFormatter(color_formatter) + + self.addHandler(console) + return \ No newline at end of file