From b426107d9ea5dccfc5aca81aa99f6509578b15a9 Mon Sep 17 00:00:00 2001 From: Luke Shingles Date: Tue, 19 Apr 2016 00:21:59 +0100 Subject: [PATCH 1/4] Fix issue 535 Treat rel. paths inside config relative to the config file import os in run_tardis() Detect and preserve absolute paths in configuration spacing replace default config_dirname with "" to have no effect on os.path.join update file paths in test configurations set config_dir on test configurations fix artis_abundances.dat path --- tardis/base.py | 5 +- tardis/io/config_reader.py | 31 +++++++-- .../data/tardis_configv1_artis_density.yml | 4 +- .../tardis_configv1_artis_density_v_slice.yml | 4 +- .../data/tardis_configv1_ascii_density.yml | 4 +- tardis/io/tests/test_config_reader.py | 64 +++++++++++-------- 6 files changed, 72 insertions(+), 40 deletions(-) diff --git a/tardis/base.py b/tardis/base.py index 83e43374282..7d080f2cbb5 100644 --- a/tardis/base.py +++ b/tardis/base.py @@ -23,11 +23,14 @@ def run_tardis(config, atom_data=None): from tardis.io import config_reader from tardis import model, simulation, atomic + import os try: config_dict = yaml.load(open(config)) + config_dirname = os.path.dirname(config) except TypeError: config_dict = config + config_dirname = '' if atom_data is not None: try: @@ -36,7 +39,7 @@ def run_tardis(config, atom_data=None): atom_data = atom_data tardis_config = config_reader.Configuration.from_config_dict( - config_dict, atom_data=atom_data) + config_dict, atom_data=atom_data, config_dirname=config_dirname) radial1d_mdl = model.Radial1DModel(tardis_config) simulation.run_radial1d(radial1d_mdl) diff --git a/tardis/io/config_reader.py b/tardis/io/config_reader.py index 06994426765..644d5eaa467 100644 --- a/tardis/io/config_reader.py +++ b/tardis/io/config_reader.py @@ -745,11 +745,15 @@ def from_yaml(cls, fname, test_parser=False): if tardis_config_version != 'v1.0': raise ConfigurationError('Currently only tardis_config_version v1.0 supported') - return cls.from_config_dict(yaml_dict, test_parser=test_parser) + config_dirname = os.path.dirname(fname) + + return cls.from_config_dict(yaml_dict, test_parser=test_parser, + config_dirname=config_dirname) @classmethod def from_config_dict(cls, config_dict, atom_data=None, test_parser=False, - config_definition_file=None, validate=True): + config_definition_file=None, validate=True, + config_dirname=''): """ Validating and subsequently parsing a config file. @@ -797,7 +801,11 @@ def from_config_dict(cls, config_dict, atom_data=None, test_parser=False, if test_parser: atom_data = None elif 'atom_data' in validated_config_dict.keys(): - atom_data_fname = validated_config_dict['atom_data'] + if os.path.isabs(validated_config_dict['atom_data']): + atom_data_fname = validated_config_dict['atom_data'] + else: + atom_data_fname = os.path.join(config_dirname, + validated_config_dict['atom_data']) validated_config_dict['atom_data_fname'] = atom_data_fname else: raise ConfigurationError('No atom_data key found in config or command line') @@ -851,9 +859,15 @@ def from_config_dict(cls, config_dict, atom_data=None, test_parser=False, validated_config_dict['supernova']['time_explosion']).cgs elif structure_section['type'] == 'file': + if os.path.isabs(structure_section['filename']): + structure_fname = structure_section['filename'] + else: + structure_fname = os.path.join(config_dirname, + structure_section['filename']) + v_inner, v_outer, mean_densities, inner_boundary_index, \ outer_boundary_index = read_density_file( - structure_section['filename'], structure_section['filetype'], + structure_fname, structure_section['filetype'], validated_config_dict['supernova']['time_explosion'], structure_section['v_inner_boundary'], structure_section['v_outer_boundary']) @@ -893,7 +907,14 @@ def from_config_dict(cls, config_dict, atom_data=None, test_parser=False, abundances.ix[z] = float(abundances_section[element_symbol_string]) elif abundances_section['type'] == 'file': - index, abundances = read_abundances_file(abundances_section['filename'], abundances_section['filetype'], + if os.path.isabs(abundances_section['filename']): + abundances_fname = abundances_section['filename'] + else: + abundances_fname = os.path.join(config_dirname, + abundances_section['filename']) + + index, abundances = read_abundances_file(abundances_fname, + abundances_section['filetype'], inner_boundary_index, outer_boundary_index) if len(index) != no_of_shells: raise ConfigurationError('The abundance file specified has not the same number of cells' diff --git a/tardis/io/tests/data/tardis_configv1_artis_density.yml b/tardis/io/tests/data/tardis_configv1_artis_density.yml index 28c6fdc814d..8ff1f1a8af2 100644 --- a/tardis/io/tests/data/tardis_configv1_artis_density.yml +++ b/tardis/io/tests/data/tardis_configv1_artis_density.yml @@ -9,7 +9,7 @@ atom_data: kurucz_atom_pure_simple.h5 model: structure: type: file - filename: tardis/io/tests/data/artis_model.dat + filename: artis_model.dat filetype: artis abundances: @@ -37,4 +37,4 @@ montecarlo: spectrum: start: 500 angstrom stop: 20000 angstrom - num: 10000 \ No newline at end of file + num: 10000 diff --git a/tardis/io/tests/data/tardis_configv1_artis_density_v_slice.yml b/tardis/io/tests/data/tardis_configv1_artis_density_v_slice.yml index 528bd380098..cad49373d72 100644 --- a/tardis/io/tests/data/tardis_configv1_artis_density_v_slice.yml +++ b/tardis/io/tests/data/tardis_configv1_artis_density_v_slice.yml @@ -9,7 +9,7 @@ atom_data: kurucz_atom_pure_simple.h5 model: structure: type: file - filename: tardis/io/tests/data/artis_model.dat + filename: artis_model.dat filetype: artis v_inner_boundary: 9000 km/s v_outer_boundary: 20000 km/s @@ -39,4 +39,4 @@ montecarlo: spectrum: start: 500 angstrom stop: 20000 angstrom - num: 10000 \ No newline at end of file + num: 10000 diff --git a/tardis/io/tests/data/tardis_configv1_ascii_density.yml b/tardis/io/tests/data/tardis_configv1_ascii_density.yml index 87ba8bfcfce..fb55ace7b40 100644 --- a/tardis/io/tests/data/tardis_configv1_ascii_density.yml +++ b/tardis/io/tests/data/tardis_configv1_ascii_density.yml @@ -9,7 +9,7 @@ atom_data: kurucz_atom_pure_simple.h5 model: structure: type: file - filename: tardis/io/tests/data/tardis_simple_ascii_density_test.dat + filename: tardis_simple_ascii_density_test.dat filetype: simple_ascii abundances: @@ -37,4 +37,4 @@ montecarlo: spectrum: start: 500 angstrom stop: 20000 angstrom - num: 10000 \ No newline at end of file + num: 10000 diff --git a/tardis/io/tests/test_config_reader.py b/tardis/io/tests/test_config_reader.py index c652a8b1a0a..1ba8b90db95 100644 --- a/tardis/io/tests/test_config_reader.py +++ b/tardis/io/tests/test_config_reader.py @@ -134,7 +134,8 @@ def test_last_no_of_packets(): yaml_data = yaml.load(open(data_path('paper1_tardis_configv1.yml'))) del yaml_data['montecarlo']['last_no_of_packets'] config = config_reader.Configuration.from_config_dict(yaml_data, - test_parser=True) + test_parser=True, + config_dirname=data_path('')) assert (config.montecarlo.last_no_of_packets == config.montecarlo.no_of_packets) @@ -144,7 +145,7 @@ def setup(self): #general parsing of the paper config filename = 'tardis_configv1_ascii_density.yml' self.config = config_reader.Configuration.from_yaml(data_path(filename), - test_parser=True) + test_parser=True) self.yaml_data = yaml.load(open(data_path(filename))) @@ -163,7 +164,7 @@ def setup(self): #general parsing of the paper config filename = 'tardis_configv1_artis_density.yml' self.config = config_reader.Configuration.from_yaml(data_path(filename), - test_parser=True) + test_parser=True) self.yaml_data = yaml.load(open(data_path(filename))) @@ -186,11 +187,12 @@ def setup(self): self.yaml_data = yaml.load(open(data_path(filename))) self.yaml_data['model']['abundances'] = {'type': 'file', - 'filename': 'tardis/io/tests/data/artis_abundances.dat', + 'filename': 'artis_abundances.dat', 'filetype': 'artis'} self.config = config_reader.Configuration.from_config_dict(self.yaml_data, - test_parser=True) + test_parser=True, + config_dirname=data_path('')) def test_velocities(self): @@ -209,11 +211,12 @@ def setup(self): self.yaml_data = yaml.load(open(data_path(filename))) self.yaml_data['model']['abundances'] = {'type': 'file', - 'filename': 'tardis/io/tests/data/artis_abundances.dat', + 'filename': 'artis_abundances.dat', 'filetype': 'artis'} self.config = config_reader.Configuration.from_config_dict(self.yaml_data, - test_parser=True) + test_parser=True, + config_dirname=data_path('')) def test_velocities(self): @@ -234,7 +237,8 @@ def setup(self): self.yaml_data = yaml.load(open(data_path(filename))) self.config = config_reader.Configuration.from_config_dict(self.yaml_data, - test_parser=True) + test_parser=True, + config_dirname=data_path('')) def test_density(self): assert_array_almost_equal(self.config.structure.mean_densities.to(u.Unit('g / cm3')).value, @@ -250,7 +254,8 @@ def setup(self): self.yaml_data['plasma']['initial_t_inner'] = "2508 K" self.config = config_reader.Configuration.from_config_dict(self.yaml_data, - test_parser=True) + test_parser=True, + config_dirname=data_path('')) def test_initial_temperature(self): assert_almost_equal(self.config.plasma.t_inner.value, 2508) @@ -263,11 +268,12 @@ def setup(self): filename = 'tardis_configv1_ascii_density_abund.yml' self.yaml_data = yaml.load(open(data_path(filename))) - self.yaml_data['model']['structure']['filename'] = 'tardis/io/tests/data/density.dat' - self.yaml_data['model']['abundances']['filename'] = 'tardis/io/tests/data/abund.dat' - + self.yaml_data['model']['structure']['filename'] = 'density.dat' + self.yaml_data['model']['abundances']['filename'] = 'abund.dat' + self.config = config_reader.Configuration.from_config_dict(self.yaml_data, - test_parser=True) + test_parser=True, + config_dirname=data_path('')) def test_velocities(self): @@ -299,8 +305,8 @@ def test_densities(self): assert_almost_equal(self.config.structure.mean_densities[4].to(u.Unit('g/cm3')).value, 8.5733893e-11/ 13.0**3 ) assert_almost_equal(self.config.structure.mean_densities[5].to(u.Unit('g/cm3')).value, 5.3037103e-11/ 13.0**3 ) assert_almost_equal(self.config.structure.mean_densities[6].to(u.Unit('g/cm3')).value, 3.3999447e-11/ 13.0**3 ) - - + + def test_ascii_reader_power_law(): with open(data_path('tardis_configv1_density_power_law_test.yml')) as f: yaml_data = yaml.load(f) @@ -309,25 +315,26 @@ def test_ascii_reader_power_law(): t_explosion = density_data['time_0'] rho_0 = density_data['rho_0'] exponent = density_data['exponent'] - + v_inner = yaml_data['model']['structure']['velocity']['start'] v_outer = yaml_data['model']['structure']['velocity']['stop'] - my_conf = config_reader.Configuration.from_yaml(data_path('tardis_configv1_density_power_law_test.yml'),test_parser=True) + my_conf = config_reader.Configuration.from_yaml(data_path('tardis_configv1_density_power_law_test.yml'), + test_parser=True) structure = my_conf['structure'] - + expected_densites = [3.29072513e-14, 2.70357804e-14, 2.23776573e-14, 1.86501954e-14, 1.56435277e-14, 1.32001689e-14, 1.12007560e-14, 9.55397475e-15, 8.18935779e-15, 7.05208050e-15, 6.09916083e-15, 5.29665772e-15, 4.61758699e-15, 4.04035750e-15, 3.54758837e-15, 3.12520752e-15, 2.76175961e-15, 2.44787115e-15, 2.17583442e-15, 1.93928168e-15] - + assert structure['no_of_shells'] == 20 for i, mdens in enumerate(expected_densites): assert_almost_equal(structure['mean_densities'][i].to( u.Unit('g / (cm3)')).value, mdens) - - + + def test_ascii_reader_exponential_law(): with open(data_path('tardis_configv1_density_exponential_test.yml')) as f: yaml_data = yaml.load(f) @@ -336,20 +343,21 @@ def test_ascii_reader_exponential_law(): t_explosion = density_data['time_0'] rho_0 = density_data['rho_0'] v0 = density_data['v_0'] - + v_inner = yaml_data['model']['structure']['velocity']['start'] v_outer = yaml_data['model']['structure']['velocity']['stop'] - my_conf = config_reader.Configuration.from_yaml(data_path('tardis_configv1_density_exponential_test.yml'),test_parser=True) + my_conf = config_reader.Configuration.from_yaml(data_path('tardis_configv1_density_exponential_test.yml'), + test_parser=True) structure = my_conf['structure'] - + expected_densites = [5.18114795e-14, 4.45945537e-14, 3.83828881e-14, 3.30364579e-14, 2.84347428e-14, 2.44740100e-14, 2.10649756e-14, 1.81307925e-14, 1.56053177e-14, 1.34316215e-14, 1.15607037e-14, 9.95038990e-15, 8.56437996e-15, 7.37143014e-15, 6.34464872e-15, 5.46088976e-15, 4.70023138e-15, 4.04552664e-15, 3.48201705e-15, 2.99699985e-15] expected_unit = 'g / (cm3)' - + assert structure['no_of_shells'] == 20 for i, mdens in enumerate(expected_densites): assert_almost_equal(structure['mean_densities'][i].value,mdens) assert structure['mean_densities'][i].unit == u.Unit(expected_unit) - - - + + + #write tests for inner and outer boundary indices From 33add7a2fe9d6a5fd8245e20f208f710d30f4d06 Mon Sep 17 00:00:00 2001 From: Fotis Tsamis Date: Mon, 2 May 2016 00:44:38 +0300 Subject: [PATCH 2/4] Write tests for the issue #535 fix. --- tardis/io/tests/test_config_reader.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tardis/io/tests/test_config_reader.py b/tardis/io/tests/test_config_reader.py index 1ba8b90db95..a65d988493d 100644 --- a/tardis/io/tests/test_config_reader.py +++ b/tardis/io/tests/test_config_reader.py @@ -358,6 +358,25 @@ def test_ascii_reader_exponential_law(): assert_almost_equal(structure['mean_densities'][i].value,mdens) assert structure['mean_densities'][i].unit == u.Unit(expected_unit) +class TestAbsoluteRelativeConfigFilePaths: + def setup(self): + self.original_cwd = os.getcwd() + self.config_filename = 'tardis_configv1_ascii_density_abund.yml' + + def test_relative_config_path_same_dir(self): + os.chdir(data_path('')) + config = config_reader.Configuration.from_yaml(self.config_filename, test_parser=True) + os.chdir(self.original_cwd) + + def test_relative_config_path_parent_dir(self): + os.chdir(os.path.join(data_path(''), os.path.pardir)) + config_path = os.path.relpath(data_path(self.config_filename)) + config = config_reader.Configuration.from_yaml(config_path, test_parser=True) + os.chdir(self.original_cwd) + + def test_absolute_config_path(self): + config = config_reader.Configuration.from_yaml(os.path.abspath(data_path(self.config_filename)), + test_parser=True) #write tests for inner and outer boundary indices From 62d69126d717556dd556432b5ea417de2db3a1e4 Mon Sep 17 00:00:00 2001 From: Fotis Tsamis Date: Mon, 2 May 2016 17:03:56 +0300 Subject: [PATCH 3/4] Empty commit to trigger a Travis CI build From f69e95b67fe489618df1ff665c19900f05f2c398 Mon Sep 17 00:00:00 2001 From: Fotis Tsamis Date: Tue, 3 May 2016 02:11:23 +0300 Subject: [PATCH 4/4] Change some relative paths to absolute in order to avoid Travis failing the tests. --- tardis/io/config_reader.py | 2 +- tardis/io/tests/test_config_reader.py | 29 ++++++++++++++++++--------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/tardis/io/config_reader.py b/tardis/io/config_reader.py index 644d5eaa467..269bd889283 100644 --- a/tardis/io/config_reader.py +++ b/tardis/io/config_reader.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) -data_dir = os.path.join(tardis.__path__[0], 'data') +data_dir = os.path.abspath(os.path.join(tardis.__path__[0], 'data')) default_config_definition_file = os.path.join(data_dir, 'tardis_config_definition.yml') diff --git a/tardis/io/tests/test_config_reader.py b/tardis/io/tests/test_config_reader.py index a65d988493d..dbe3dde2030 100644 --- a/tardis/io/tests/test_config_reader.py +++ b/tardis/io/tests/test_config_reader.py @@ -1,6 +1,7 @@ # tests for the config reader module from tardis.io import config_reader from astropy import units as u +from contextlib import contextmanager import os import pytest import yaml @@ -11,7 +12,20 @@ def data_path(filename): data_dir = os.path.dirname(__file__) - return os.path.join(data_dir, 'data', filename) + return os.path.abspath(os.path.join(data_dir, 'data', filename)) + +@contextmanager +def change_cwd(directory): + # Function taken from: + # https://github.com/django/django/blob/18d962f2e6cc8829d60d6f6dfb3ee3855fa5362e/tests/test_runner/test_discover_runner.py + current_dir = os.path.abspath(os.path.dirname(__file__)) + new_dir = os.path.join(current_dir, directory) + old_cwd = os.getcwd() + os.chdir(new_dir) + try: + yield + finally: + os.chdir(old_cwd) def test_config_namespace_attribute_test(): namespace = config_reader.ConfigurationNameSpace({'param1':1}) @@ -360,19 +374,16 @@ def test_ascii_reader_exponential_law(): class TestAbsoluteRelativeConfigFilePaths: def setup(self): - self.original_cwd = os.getcwd() self.config_filename = 'tardis_configv1_ascii_density_abund.yml' def test_relative_config_path_same_dir(self): - os.chdir(data_path('')) - config = config_reader.Configuration.from_yaml(self.config_filename, test_parser=True) - os.chdir(self.original_cwd) + with change_cwd(data_path('')): + config = config_reader.Configuration.from_yaml(self.config_filename, test_parser=True) def test_relative_config_path_parent_dir(self): - os.chdir(os.path.join(data_path(''), os.path.pardir)) - config_path = os.path.relpath(data_path(self.config_filename)) - config = config_reader.Configuration.from_yaml(config_path, test_parser=True) - os.chdir(self.original_cwd) + config_path = os.path.relpath(data_path(self.config_filename), data_path(os.path.pardir)) + with change_cwd(data_path(os.path.pardir)): + config = config_reader.Configuration.from_yaml(config_path, test_parser=True) def test_absolute_config_path(self): config = config_reader.Configuration.from_yaml(os.path.abspath(data_path(self.config_filename)),