Merge pull request #1815 from mrmundt/pyutilib-misc

Replace pyutilib.misc.import_file

examples/bilevel/test1.py

@@ -10,8 +10,8 @@
 
 from pyomo.environ import *
 import sys
-import pyutilib.misc
-pyutilib.misc.import_file(sys.argv[1])
+from pyomo.common.fileutils import import_file
+import_file(sys.argv[1])
 
 xfrm = TransformationFactory('bilevel.linear_dual')
 model2 = xfrm.apply(example.pyomo_create_model(None,None))

examples/pyomo/benders/runbenders

@@ -6,9 +6,8 @@
 #
 
 # Python imports
-from pyutilib.misc import import_file
 from pyomo.environ import *
-from pyomo.opt import SolverFactory
+from pyomo.common.fileutils import import_file
 from pyomo.opt.base import SolverFactory
 from pyomo.opt.parallel import SolverManagerFactory
 from pyomo.opt.parallel.manager import solve_all_instances

pyomo/bilevel/tests/test_blp.py

@@ -18,9 +18,9 @@
 exdir = normpath(join(currdir,'..','..','..','examples','bilevel'))
 
 import pyutilib.th as unittest
-import pyutilib.misc
 
 from pyomo.common.dependencies import yaml, yaml_available, yaml_load_args
+from pyomo.common.fileutils import import_file
 import pyomo.opt
 import pyomo.scripting.pyomo_main as pyomo_main
 from pyomo.scripting.util import cleanup
@@ -116,7 +116,7 @@ def tearDown(self):
             os.remove(os.path.join(currdir,'result.yml'))
 
     def run_bilevel(self,  *args, **kwds):
-        module = pyutilib.misc.import_file(args[0])
+        module = import_file(args[0])
         instance = module.pyomo_create_model(None, None)
         xfrm = TransformationFactory('bilevel.linear_mpec')
         xfrm.apply_to(instance, deterministic=True)

pyomo/common/fileutils.py

@@ -15,6 +15,8 @@
 import os
 import platform
 import six
+import importlib.util
+import sys
 
 from .deprecation import deprecated
 from . import config
@@ -388,6 +390,31 @@ def find_executable(exename, cwd=True, include_PATH=True, pathlist=None):
                      pathlist=pathlist, allow_pathlist_deep_references=False)
 
 
+def import_file(path, clear_cache=False):
+    """
+    Import a module given the full path/filename of the file.
+    Replaces import_file from pyutilib (Pyomo 6.0.0).
+    
+    This function returns the module object that is created.
+    Parameters
+    ----------
+    path : str
+        Full path to .py file.
+    clear_cache: bool
+        Remove module if already loaded. The default is False.
+    """
+    path = os.path.expanduser(os.path.expandvars(path))
+    if not os.path.exists(path):
+        raise FileNotFoundError('File does not exist. Check path.')
+    module_dir, module_file = os.path.split(path)
+    module_name, module_ext = os.path.splitext(module_file)
+    if clear_cache and module_name in sys.modules:
+        del sys.modules[module_name]
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    module = spec.loader.load_module()
+    return module
+
+
 class _PathData(object):
     def __init__(self, manager, name):
         self._mngr = manager

pyomo/common/tests/import_ex.py

@@ -0,0 +1,4 @@
+def a():
+    pass
+
+b = 2

pyomo/common/tests/test_fileutils.py

@@ -27,6 +27,7 @@
 from pyomo.common.fileutils import (
     this_file, this_file_dir, find_file, find_library, find_executable, 
     PathManager, _system, _path, _exeExt, _libExt, _ExecutableData,
+    import_file
 )
 from pyomo.common.download import FileDownloader
 
@@ -99,6 +100,23 @@ def test_this_file_dir(self):
         expected_path = os.path.join('pyomo','common','tests')
         self.assertTrue(_this_file_dir.endswith(expected_path))
 
+    def test_import_file(self):
+        import_ex = import_file(os.path.join(_this_file_dir, 'import_ex.py'))
+        if not "import_ex" in sys.modules.keys():
+            self.fail("test_import_file - failed to import the import_ex.py file")
+
+    def test_import_vars(self):
+        import_ex = import_file(os.path.join(_this_file_dir, 'import_ex.py'))
+        try:
+            importvar = import_ex.a
+        except:
+            self.fail('test_import_vars - failed to access data in import_ex.py file.')
+
+    def test_import_file_no_extension(self):
+        with self.assertRaises(FileNotFoundError) as context:
+            import_file(os.path.join(_this_file_dir, 'import_ex'))
+        self.assertTrue('File does not exist' in str(context.exception))
+
     def test_system(self):
         self.assertTrue(platform.system().lower().startswith(_system()))
         self.assertNotIn('.', _system())

pyomo/contrib/gdpopt/tests/test_LBB.py

@@ -4,8 +4,8 @@
 from os.path import abspath, dirname, join, normpath
 
 import pyutilib.th as unittest
-from pyutilib.misc import import_file
 
+from pyomo.common.fileutils import import_file
 from pyomo.contrib.satsolver.satsolver import z3_available
 from pyomo.environ import SolverFactory, value, ConcreteModel, Var, Objective, maximize
 from pyomo.gdp import Disjunction

pyomo/contrib/gdpopt/tests/test_gdpopt.py

@@ -18,13 +18,13 @@
 import pyutilib.th as unittest
 from pyomo.common.log import LoggingIntercept
 from pyomo.common.collections import Container
+from pyomo.common.fileutils import import_file
 from pyomo.contrib.gdpopt.GDPopt import GDPoptSolver
 from pyomo.contrib.gdpopt.data_class import GDPoptSolveData
 from pyomo.contrib.gdpopt.mip_solve import solve_linear_GDP
 from pyomo.contrib.gdpopt.util import is_feasible, time_code
 from pyomo.environ import ConcreteModel, Objective, SolverFactory, Var, value, Integers, Block, Constraint, maximize
 from pyomo.gdp import Disjunct, Disjunction
-from pyutilib.misc import import_file
 from pyomo.contrib.mcpp.pyomo_mcpp import mcpp_available
 from pyomo.opt import TerminationCondition
 

pyomo/contrib/pynumero/examples/external_grey_box/react-example/maximize_cb_outputs.py → pyomo/contrib/pynumero/examples/external_grey_box/react_example/maximize_cb_outputs.py

@@ -12,7 +12,7 @@
 import pyomo.environ as pyo
 from pyomo.contrib.pynumero.interfaces.external_grey_box import \
     ExternalGreyBoxBlock
-from reactor_model_outputs import ReactorConcentrationsOutputModel
+from pyomo.contrib.pynumero.examples.external_grey_box.react_example.reactor_model_outputs import ReactorConcentrationsOutputModel
 
 def maximize_cb_outputs(show_solver_log=False):
     # in this simple example, we will use an external grey box model representing

pyomo/contrib/pynumero/examples/external_grey_box/react-example/maximize_cb_ratio_residuals.py → pyomo/contrib/pynumero/examples/external_grey_box/react_example/maximize_cb_ratio_residuals.py

@@ -7,9 +7,10 @@
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
+
 import pyomo.environ as pyo
 from pyomo.contrib.pynumero.interfaces.external_grey_box import ExternalGreyBoxBlock
-from reactor_model_residuals import ReactorModel, ReactorModelWithHessian, \
+from pyomo.contrib.pynumero.examples.external_grey_box.react_example.reactor_model_residuals import ReactorModel, ReactorModelWithHessian, \
     ReactorModelNoOutputs, ReactorModelScaled, create_pyomo_reactor_model
 
 def maximize_cb_ratio_residuals_with_output(show_solver_log=False, additional_options={}):

pyomo/contrib/pynumero/tests/test_cyipopt_examples.py

@@ -1,7 +1,16 @@
+#  ___________________________________________________________________________
+#
+#  Pyomo: Python Optimization Modeling Objects
+#  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
+#  Under the terms of Contract DE-NA0003525 with National Technology and 
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  rights in this software.
+#  This software is distributed under the 3-clause BSD License.
+#  ___________________________________________________________________________
+
 import os.path
-from pyomo.common.fileutils import this_file_dir
+from pyomo.common.fileutils import this_file_dir, import_file
 import pyutilib.th as unittest
-from pyutilib.misc import import_file
 import pyomo.environ as pyo
 
 from pyomo.contrib.pynumero.dependencies import (
@@ -36,13 +45,13 @@ def test_status_maps(self):
 
 class TestExamples(unittest.TestCase):
     def test_external_grey_box_react_example_maximize_cb_outputs(self):
-        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react-example', 'maximize_cb_outputs.py'))
+        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react_example', 'maximize_cb_outputs.py'))
         m = ex.maximize_cb_outputs()
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['sv']), 1.34381, places=3)
         self.assertAlmostEqual(pyo.value(m.reactor.outputs['cb']), 1072.4372, places=2)
 
     def test_external_grey_box_react_example_maximize_cb_outputs_scaling(self):
-        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react-example', 'maximize_cb_ratio_residuals.py'))
+        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react_example', 'maximize_cb_ratio_residuals.py'))
         aoptions={'nlp_scaling_method': 'user-scaling',
                  'output_file': '_cyipopt-external-greybox-react-scaling.log',
                  'file_print_level':10}
@@ -78,35 +87,35 @@ def test_external_grey_box_react_example_maximize_cb_outputs_scaling(self):
         self.assertIn('c scaling vector[    6]= 1.0000000000000000e+01', solver_trace)
 
     def test_external_grey_box_react_example_maximize_with_output(self):
-        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react-example', 'maximize_cb_ratio_residuals.py'))
+        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react_example', 'maximize_cb_ratio_residuals.py'))
         m = ex.maximize_cb_ratio_residuals_with_output()
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['sv']), 1.26541996, places=3)
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['cb']), 1071.7410089, places=2)
         self.assertAlmostEqual(pyo.value(m.reactor.outputs['cb_ratio']), 0.15190409266, places=3)
 
     def test_external_grey_box_react_example_maximize_with_hessian_with_output(self):
-        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react-example', 'maximize_cb_ratio_residuals.py'))
+        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react_example', 'maximize_cb_ratio_residuals.py'))
         m = ex.maximize_cb_ratio_residuals_with_hessian_with_output()
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['sv']), 1.26541996, places=3)
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['cb']), 1071.7410089, places=2)
         self.assertAlmostEqual(pyo.value(m.reactor.outputs['cb_ratio']), 0.15190409266, places=3)
 
     def test_external_grey_box_react_example_maximize_with_hessian_with_output_pyomo(self):
-        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react-example', 'maximize_cb_ratio_residuals.py'))
+        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react_example', 'maximize_cb_ratio_residuals.py'))
         m = ex.maximize_cb_ratio_residuals_with_hessian_with_output_pyomo()
         self.assertAlmostEqual(pyo.value(m.sv), 1.26541996, places=3)
         self.assertAlmostEqual(pyo.value(m.cb), 1071.7410089, places=2)
         self.assertAlmostEqual(pyo.value(m.cb_ratio), 0.15190409266, places=3)
 
     def test_pyomo_react_example_maximize_with_obj(self):
-        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react-example', 'maximize_cb_ratio_residuals.py'))
+        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react_example', 'maximize_cb_ratio_residuals.py'))
         m = ex.maximize_cb_ratio_residuals_with_obj()
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['sv']), 1.26541996, places=3)
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['cb']), 1071.7410089, places=2)
         self.assertAlmostEqual(pyo.value(m.obj), 0.15190409266, places=3)
 
     def test_external_grey_box_react_example_maximize_with_additional_pyomo_variables(self):
-        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react-example', 'maximize_cb_ratio_residuals.py'))
+        ex = import_file(os.path.join(example_dir, 'external_grey_box', 'react_example', 'maximize_cb_ratio_residuals.py'))
         m = ex.maximize_cb_ratio_residuals_with_pyomo_variables()
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['sv']), 1.26541996, places=3)
         self.assertAlmostEqual(pyo.value(m.reactor.inputs['cb']), 1071.7410089, places=2)

pyomo/contrib/satsolver/test_satsolver.py

@@ -1,8 +1,18 @@
+#  ___________________________________________________________________________
+#
+#  Pyomo: Python Optimization Modeling Objects
+#  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
+#  Under the terms of Contract DE-NA0003525 with National Technology and 
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  rights in this software.
+#  This software is distributed under the 3-clause BSD License.
+#  ___________________________________________________________________________
+
 from os.path import abspath, dirname, join, normpath
 
 import pyutilib.th as unittest
-from pyutilib.misc import import_file
 
+from pyomo.common.fileutils import import_file
 from pyomo.contrib.satsolver.satsolver import satisfiable, z3_available
 from pyomo.core.base.set_types import PositiveIntegers, NonNegativeReals, Binary
 from pyomo.environ import (

pyomo/dae/tests/test_simulator.py

@@ -33,10 +33,10 @@
     IndexTemplate, 
     _GetItemIndexer,
 )
+from pyomo.common.fileutils import import_file
 
 import os
 from pyutilib.misc import setup_redirect, reset_redirect
-from pyutilib.misc import import_file
 
 from os.path import abspath, dirname, normpath, join
 currdir = dirname(abspath(__file__))

pyomo/gdp/tests/test_basic_step.py

@@ -1,12 +1,22 @@
+#  ___________________________________________________________________________
+#
+#  Pyomo: Python Optimization Modeling Objects
+#  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
+#  Under the terms of Contract DE-NA0003525 with National Technology and 
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  rights in this software.
+#  This software is distributed under the 3-clause BSD License.
+#  ___________________________________________________________________________
+
+
 """Tests for applying basic steps."""
 import pyutilib.th as unittest
 from pyomo.core import Constraint, Var, SortComponents
 from pyomo.gdp.basic_step import apply_basic_step
 from pyomo.repn import generate_standard_repn
 import pyomo.gdp.tests.models as models
 import pyomo.gdp.tests.common_tests as ct
-
-from pyutilib.misc import import_file
+from pyomo.common.fileutils import import_file
 
 from os.path import abspath, dirname, normpath, join
 currdir = dirname(abspath(__file__))

pyomo/gdp/tests/test_gdp.py

@@ -15,7 +15,7 @@
 import os
 import sys
 from os.path import abspath, dirname, normpath, join
-from pyutilib.misc import import_file
+from pyomo.common.fileutils import import_file
 currdir = dirname(abspath(__file__))
 exdir = normpath(join(currdir,'..','..','..','examples', 'gdp'))
 

pyomo/scripting/util.py

@@ -22,8 +22,9 @@
 from pyomo.common.deprecation import deprecated
 from pyomo.common.log import is_debug_set
 from pyomo.common.tempfiles import TempfileManager
+from pyomo.common.fileutils import import_file
 
-from pyutilib.misc import import_file, setup_redirect, reset_redirect
+from pyutilib.misc import setup_redirect, reset_redirect
 
 from pyomo.common.dependencies import (
     yaml, yaml_available, yaml_load_args,
@@ -188,7 +189,8 @@ def apply_preprocessing(data, parser=None):
     #
     filter_excepthook=True
     tick = time.time()
-    data.local.usermodel = import_file(data.options.model.filename, clear_cache=True)
+    data.local.usermodel = import_file(data.options.model.filename,
+                                       clear_cache=True)
     data.local.time_initial_import = time.time()-tick
     filter_excepthook=False
 
@@ -350,7 +352,8 @@ def create_model(data):
                                                  profile_memory=data.options.runtime.profile_memory,
                                                  report_timing=data.options.runtime.report_timing)
             elif suffix == "py":
-                userdata = import_file(data.options.data.files[0], clear_cache=True)
+                userdata = import_file(data.options.data.files[0],
+                                       clear_cache=True)
                 if "modeldata" in dir(userdata):
                     if len(ep) == 1:
                         msg = "Cannot apply 'pyomo_create_modeldata' and use the" \
@@ -502,7 +505,7 @@ def apply_optimizer(data, instance=None):
     if len(data.options.solvers[0].suffixes) > 0:
         for suffix_name in data.options.solvers[0].suffixes:
             if suffix_name[0] in ['"',"'"]:
-                suffix_name = suffix[1:-1]
+                suffix_name = suffix_name[1:-1]
             # Don't redeclare the suffix if it already exists
             suffix = getattr(instance, suffix_name, None)
             if suffix is None:

pyomo/solvers/tests/piecewise_linear/test_piecewise_linear.py

@@ -13,10 +13,10 @@
 thisDir = dirname( abspath(__file__) )
 
 import pyutilib.th as unittest
-import pyutilib.misc
 
 import pyomo.opt
 from pyomo.common.dependencies import yaml, yaml_available, yaml_load_args
+from pyomo.common.fileutils import import_file
 from pyomo.core.base import Var
 from pyomo.core.base.objective import minimize, maximize
 from pyomo.core.base.piecewise import Bound, PWRepn
@@ -53,10 +53,8 @@ def testMethod(obj):
             obj.skipTest("Solver %s (interface=%s) is not available"
                          % (solver, writer))
 
-        m = pyutilib.misc.import_file(os.path.join(thisDir,
-                                                   'problems',
-                                                   problem),
-                                      clear_cache=True)
+        m = import_file(os.path.join(thisDir, 'problems', problem + '.py'),
+                        clear_cache=True)
 
         model = m.define_model(**kwds)
 

pyomo/solvers/tests/piecewise_linear/test_piecewise_linear_kernel.py

@@ -14,8 +14,8 @@
 thisDir = dirname( abspath(__file__) )
 
 import pyutilib.th as unittest
-from pyutilib.misc import import_file
 
+from pyomo.common.fileutils import import_file
 from pyomo.kernel import SolverFactory, variable, maximize, minimize
 from pyomo.solvers.tests.solvers import test_solver_cases
 
@@ -43,10 +43,8 @@ def testMethod(obj):
             obj.skipTest("Solver %s (interface=%s) is not available"
                          % (solver, writer))
 
-        m = import_file(os.path.join(thisDir,
-                                                   'kernel_problems',
-                                                   problem),
-                                      clear_cache=True)
+        m = import_file(os.path.join(thisDir, 'kernel_problems', problem + '.py'),
+                        clear_cache=True)
 
         model = m.define_model(**kwds)