upgraded to importlib

conans/client/hook_manager.py

@@ -1,12 +1,9 @@
 import os
-import sys
-import traceback
-import uuid
 from collections import defaultdict
 from threading import Lock
 
 from conans.cli.output import ScopedOutput, ConanOutput
-from conans.client.tools.files import chdir
+from conans.client.loader import load_python_file
 from conans.errors import ConanException, NotFoundException
 
 valid_hook_methods = ["pre_export", "post_export",
@@ -34,6 +31,7 @@ def __init__(self, hooks_folder, hook_names):
     def execute(self, method_name, **kwargs):
         # It is necessary to protect the lazy loading of hooks with a mutex, because it can be
         # concurrent (e.g. upload --parallel)
+        # TODO: This reads a bit insane, simplify it?
         self._mutex.acquire()
         try:
             if not self.hooks:
@@ -59,7 +57,7 @@ def _load_hook(self, hook_name):
             hook_name = "%s.py" % hook_name
         hook_path = os.path.normpath(os.path.join(self._hooks_folder, hook_name))
         try:
-            hook = HookManager._load_module_from_file(hook_path)
+            hook, _ = load_python_file(hook_path)
             for method in valid_hook_methods:
                 hook_method = getattr(hook, method, None)
                 if hook_method:
@@ -70,48 +68,3 @@ def _load_hook(self, hook_name):
                                                                          self._hooks_folder))
         except Exception as e:
             raise ConanException("Error loading hook '%s': %s" % (hook_path, str(e)))
-
-    @staticmethod
-    def _load_module_from_file(hook_path):
-        """ From a given path, obtain the in memory python import module
-        """
-        if not os.path.exists(hook_path):
-            raise NotFoundException
-        filename = os.path.splitext(os.path.basename(hook_path))[0]
-        current_dir = os.path.dirname(hook_path)
-
-        old_dont_write_bytecode = sys.dont_write_bytecode
-        try:
-            sys.path.append(current_dir)
-            old_modules = list(sys.modules.keys())
-            with chdir(current_dir):
-                sys.dont_write_bytecode = True
-                loaded = __import__(filename)
-            # Put all imported files under a new package name
-            module_id = uuid.uuid1()
-            added_modules = set(sys.modules).difference(old_modules)
-            for added in added_modules:
-                module = sys.modules[added]
-                if module:
-                    try:
-                        try:
-                            # Most modules will have __file__ != None
-                            folder = os.path.dirname(module.__file__)
-                        except (AttributeError, TypeError):
-                            # But __file__ might not exist or equal None
-                            # Like some builtins and Namespace packages py3
-                            folder = module.__path__._path[0]
-                    except AttributeError:  # In case the module.__path__ doesn't exist
-                        pass
-                    else:
-                        if folder.startswith(current_dir):
-                            module = sys.modules.pop(added)
-                            sys.modules["%s.%s" % (module_id, added)] = module
-        except Exception:
-            trace = traceback.format_exc().split('\n')
-            raise ConanException("Unable to load Hook in %s\n%s" % (hook_path,
-                                                                    '\n'.join(trace[3:])))
-        finally:
-            sys.dont_write_bytecode = old_dont_write_bytecode
-            sys.path.pop()
-        return loaded

conans/client/loader.py

@@ -1,4 +1,4 @@
-import imp
+from importlib import invalidate_caches, util as imp_util
 import inspect
 import os
 import sys
@@ -9,8 +9,7 @@
 from conans.client.conf.required_version import validate_conan_version
 from conans.client.loader_txt import ConanFileTextLoader
 from conans.client.tools.files import chdir
-from conans.errors import ConanException, NotFoundException, ConanInvalidConfiguration, \
-    conanfile_exception_formatter
+from conans.errors import ConanException, NotFoundException, conanfile_exception_formatter
 from conans.model.conan_file import ConanFile
 from conans.model.options import Options
 from conans.model.recipe_ref import RecipeReference
@@ -25,6 +24,7 @@ def __init__(self, runner, pyreq_loader=None, requester=None):
         self._pyreq_loader = pyreq_loader
         self._cached_conanfile_classes = {}
         self._requester = requester
+        invalidate_caches()
 
     def load_basic(self, conanfile_path, graph_lock=None, display=""):
         """ loads a conanfile basic object without evaluating anything
@@ -83,7 +83,7 @@ def load_generators(self, conanfile_path):
             to the provided generator list
             @param conanfile_module: the module to be processed
             """
-        conanfile_module, module_id = _parse_conanfile(conanfile_path)
+        conanfile_module, module_id = load_python_file(conanfile_path)
         for name, attr in conanfile_module.__dict__.items():
             if (name.startswith("_") or not inspect.isclass(attr) or
                     attr.__dict__.get("__module__") != module_id):
@@ -294,15 +294,15 @@ class defining the Recipe, but also process possible existing generators
 
 
 def parse_conanfile(conanfile_path):
-    module, filename = _parse_conanfile(conanfile_path)
+    module, filename = load_python_file(conanfile_path)
     try:
         conanfile = _parse_module(module, filename)
         return module, conanfile
     except Exception as e:  # re-raise with file name
         raise ConanException("%s: %s" % (conanfile_path, str(e)))
 
 
-def _parse_conanfile(conan_file_path):
+def load_python_file(conan_file_path):
     """ From a given path, obtain the in memory python import module
     """
 
@@ -317,7 +317,9 @@ def _parse_conanfile(conan_file_path):
         with chdir(current_dir):
             old_dont_write_bytecode = sys.dont_write_bytecode
             sys.dont_write_bytecode = True
-            loaded = imp.load_source(module_id, conan_file_path)
+            spec = imp_util.spec_from_file_location(module_id, conan_file_path)
+            loaded = imp_util.module_from_spec(spec)
+            spec.loader.exec_module(loaded)
             sys.dont_write_bytecode = old_dont_write_bytecode
 
         required_conan_version = getattr(loaded, "required_conan_version", None)

conans/test/unittests/client/conanfile_loader_test.py

@@ -2,19 +2,15 @@
 import sys
 import textwrap
 import unittest
-from collections import OrderedDict
 
 import pytest
 from mock import Mock, call
 from parameterized import parameterized
 
-from conans.client.loader import ConanFileLoader, ConanFileTextLoader, _parse_conanfile
+from conans.client.loader import ConanFileLoader, ConanFileTextLoader, load_python_file
 from conans.client.tools.files import chdir
 from conans.errors import ConanException
 from conans.model.options import Options
-from conans.model.profile import Profile
-from conans.model.requires import Requirements
-from conans.model.settings import Settings
 from conans.test.utils.test_files import temp_folder
 from conans.test.utils.tools import create_profile
 from conans.util.files import save
@@ -234,7 +230,7 @@ def conanfile_func():
                 save("__init__.py", "")
                 save("{}/__init__.py".format(subdir_name), "")
 
-        loaded, module_id = _parse_conanfile(os.path.join(tmp, "conanfile.py"))
+        loaded, module_id = load_python_file(os.path.join(tmp, "conanfile.py"))
         return loaded, module_id, expected_return
 
     @parameterized.expand([(True, False), (False, True), (False, False)])
@@ -288,8 +284,8 @@ def append(data):
 
         try:
             sys.path.append(temp)
-            loaded1, _ = _parse_conanfile(os.path.join(temp1, "conanfile.py"))
-            loaded2, _ = _parse_conanfile(os.path.join(temp2, "conanfile.py"))
+            loaded1, _ = load_python_file(os.path.join(temp1, "conanfile.py"))
+            loaded2, _ = load_python_file(os.path.join(temp2, "conanfile.py"))
             self.assertIs(loaded1.myconanlogger, loaded2.myconanlogger)
             self.assertIs(loaded1.myconanlogger.value, loaded2.myconanlogger.value)
         finally: