From 6125dd356df8f86f457d188a4bcec14ecf733116 Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 16 Jun 2017 10:30:25 +0900 Subject: [PATCH 01/28] WIP beginning of custom importer implementation --- pyros_msgs/importer/rosmsg_importer.py | 316 ++++++++++++++++-- pyros_msgs/importer/tests/srv/TestSrv.srv | 4 + .../tests/test_rosmsg_generator_import.py | 101 +++++- 3 files changed, 382 insertions(+), 39 deletions(-) create mode 100644 pyros_msgs/importer/tests/srv/TestSrv.srv diff --git a/pyros_msgs/importer/rosmsg_importer.py b/pyros_msgs/importer/rosmsg_importer.py index 6a32022..fd801c3 100644 --- a/pyros_msgs/importer/rosmsg_importer.py +++ b/pyros_msgs/importer/rosmsg_importer.py @@ -1,5 +1,9 @@ from __future__ import absolute_import, division, print_function +import imp + +import py + """ A module to setup custom importer for .msg and .srv files Upon import, it will first find the .msg file, then generate the python module for it, then load it. @@ -11,7 +15,7 @@ # Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module # Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path -# Note : Coudlnt find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) +# Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) import os import sys @@ -19,6 +23,8 @@ import genpy.generator import genpy.generate_initpy +import logging + class ROSLoader(object): def __init__(self, generator, ext): @@ -38,33 +44,297 @@ def load_module(self, fullname): return +# https://pymotw.com/2/sys/imports.html#sys-imports +class NoisyImportFinder(object): + PATH_TRIGGER = 'NoisyImportFinder_PATH_TRIGGER' + + def __init__(self, path_entry): + print('Checking NoisyImportFinder support for %s' % path_entry) + if path_entry != self.PATH_TRIGGER: + print('NoisyImportFinder does not work for %s' % path_entry) + raise ImportError() + return + + def find_module(self, fullname, path=None): + print('NoisyImportFinder looking for "%s"' % fullname) + return None + +# https://pymotw.com/3/sys/imports.html#sys-imports + +# class NoisyImportFinder: +# +# PATH_TRIGGER = 'NoisyImportFinder_PATH_TRIGGER' +# +# def __init__(self, path_entry): +# print('Checking {}:'.format(path_entry), end=' ') +# if path_entry != self.PATH_TRIGGER: +# print('wrong finder') +# raise ImportError() +# else: +# print('works') +# return +# +# def find_module(self, fullname, path=None): +# print('Looking for {!r}'.format(fullname)) +# return None +# +# +# sys.path_hooks.append(NoisyImportFinder) + + + + + +if sys.version_info >= (3, ): + class ROSImportFinder(object): + + PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' + + def __init__(self, path_entry): + + self.logger = logging.getLogger(__name__) + self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) + if path_entry != self.PATH_TRIGGER: + self.logger.debug('ROSImportFinder does not work for %s' % path_entry) + raise ImportError() + + self.loaders = { + '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), + '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') + } + + def find_module(self, fullname, path=None): + print('ROSImportFinder looking for "%s"' % fullname) + return None + +elif sys.version_info >= (2, 7, 12): + class ROSImportFinder(object): + PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' + + def __init__(self, path_entry): + self.logger = logging.getLogger(__name__) + self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) + if path_entry != self.PATH_TRIGGER: + self.logger.debug('ROSImportFinder does not work for %s' % path_entry) + raise ImportError() + + self.loaders = { + '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), + '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') + } + + def find_module(self, fullname, path=None): + print('ROSImportFinder looking for "%s"' % fullname) + return None + +else: + class ROSImportFinder(object): + + PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' + + def __init__(self, path_entry=None): + self.logger = logging.getLogger(__name__) + self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) + if path_entry is None: # called on very old python (< 2.7.12) + pass + elif path_entry != self.PATH_TRIGGER: # python following correct PEP ( 302 ?) + self.logger.debug('ROSImportFinder does not work for %s' % path_entry) + raise ImportError() + + self.loaders = { + '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), + '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') + } + + def find_module(self, name, path=None): + self.logger.debug('ROSImportFinder looking for "%s"' % name) + + # implementation inspired from pytest.rewrite + names = name.rsplit(".", 1) + lastname = names[-1] + pth = None + if path is not None: + # Starting with Python 3.3, path is a _NamespacePath(), which + # causes problems if not converted to list. + path = list(path) + if len(path) == 1: + pth = path[0] + + + if pth is None: + + + try: + fd, fn, desc = imp.find_module(lastname, path) + except ImportError: + return None + if fd is not None: + fd.close() + + + + + tp = desc[2] + if tp == imp.PY_COMPILED: + if hasattr(imp, "source_from_cache"): + try: + fn = imp.source_from_cache(fn) + except ValueError: + # Python 3 doesn't like orphaned but still-importable + # .pyc files. + fn = fn[:-1] + else: + fn = fn[:-1] + elif tp != imp.PY_SOURCE: + # Don't know what this is. + return None + else: + fn = os.path.join(pth, name.rpartition(".")[2] + ".py") + + # fn_pypath = py.path.local(fn) + # if not self._should_rewrite(name, fn_pypath, state): + # return None + # + # self._rewritten_names.add(name) + + # def find_module(self, name, path=None): + # """ + # Return the loader for the specified module. + # """ + # # Ref : https://www.python.org/dev/peps/pep-0302/#specification-part-1-the-importer-protocol + # + # # + # loader = None + # + # # path = path or sys.path + # # for p in path: + # # for f in os.listdir(p): + # # filename, ext = os.path.splitext(f) + # # # our modules generated from messages are always a leaf in import tree so we only care about this case + # # if ext in self.loaders.keys() and filename == name.split('.')[-1]: + # # loader = self.loaders.get(ext) + # # break # we found it. break out. + # # + # # return loader + # + # # implementation inspired from pytest.rewrite + # self.logger.debug("find_module called for: %s" % name) + # names = name.rsplit(".", 1) + # lastname = names[-1] + # pth = None + # if path is not None: + # # Starting with Python 3.3, path is a _NamespacePath(), which + # # causes problems if not converted to list. + # path = list(path) + # if len(path) == 1: + # pth = path[0] + # + # + # if pth is None: + # + # + # + # + # + # try: + # fd, fn, desc = imp.find_module(lastname, path) + # except ImportError: + # return None + # if fd is not None: + # fd.close() + # + # + # + # + # tp = desc[2] + # if tp == imp.PY_COMPILED: + # if hasattr(imp, "source_from_cache"): + # try: + # fn = imp.source_from_cache(fn) + # except ValueError: + # # Python 3 doesn't like orphaned but still-importable + # # .pyc files. + # fn = fn[:-1] + # else: + # fn = fn[:-1] + # elif tp != imp.PY_SOURCE: + # # Don't know what this is. + # return None + # else: + # fn = os.path.join(pth, name.rpartition(".")[2] + ".py") + # + # fn_pypath = py.path.local(fn) + # if not self._should_rewrite(name, fn_pypath, state): + # return None + # + # self._rewritten_names.add(name) + # + # # The requested module looks like a test file, so rewrite it. This is + # # the most magical part of the process: load the source, rewrite the + # # asserts, and load the rewritten source. We also cache the rewritten + # # module code in a special pyc. We must be aware of the possibility of + # # concurrent pytest processes rewriting and loading pycs. To avoid + # # tricky race conditions, we maintain the following invariant: The + # # cached pyc is always a complete, valid pyc. Operations on it must be + # # atomic. POSIX's atomic rename comes in handy. + # write = not sys.dont_write_bytecode + # cache_dir = os.path.join(fn_pypath.dirname, "__pycache__") + # if write: + # try: + # os.mkdir(cache_dir) + # except OSError: + # e = sys.exc_info()[1].errno + # if e == errno.EEXIST: + # # Either the __pycache__ directory already exists (the + # # common case) or it's blocked by a non-dir node. In the + # # latter case, we'll ignore it in _write_pyc. + # pass + # elif e in [errno.ENOENT, errno.ENOTDIR]: + # # One of the path components was not a directory, likely + # # because we're in a zip file. + # write = False + # elif e in [errno.EACCES, errno.EROFS, errno.EPERM]: + # state.trace("read only directory: %r" % fn_pypath.dirname) + # write = False + # else: + # raise + # cache_name = fn_pypath.basename[:-3] + PYC_TAIL + # pyc = os.path.join(cache_dir, cache_name) + # # Notice that even if we're in a read-only directory, I'm going + # # to check for a cached pyc. This may not be optimal... + # co = _read_pyc(fn_pypath, pyc, state.trace) + # if co is None: + # state.trace("rewriting %r" % (fn,)) + # source_stat, co = _rewrite_test(self.config, fn_pypath) + # if co is None: + # # Probably a SyntaxError in the test. + # return None + # if write: + # _make_rewritten_pyc(state, source_stat, pyc, co) + # else: + # state.trace("found cached rewritten pyc for %r" % (fn,)) + # self.modules[name] = co, pyc + # return self -class ROSFinder(object): +_ros_finder_instance_obsolete_python = ROSImportFinder - def __init__(self): - self.loaders = { - '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), - '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') - } - def find_module(self, name, path=None): - """ - Return the loader for the specified module. - """ - # Ref : https://www.python.org/dev/peps/pep-0302/#specification-part-1-the-importer-protocol +def activate(): + if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? + sys.path_hooks.append(ROSImportFinder) + else: # older (trusty) version + sys.path_hooks.append(_ros_finder_instance_obsolete_python) - loader = None + for hook in sys.path_hooks: + print('Path hook: {}'.format(hook)) - path = path or sys.path - for p in path: - for f in os.listdir(p): - filename, ext = os.path.splitext(f) - # our modules generated from messages are always a leaf in import tree so we only care about this case - if ext in self.loaders.keys() and filename == name.split('.')[-1]: - loader = self.loaders.get(ext) - break # we found it. break out. + sys.path.insert(0, ROSImportFinder.PATH_TRIGGER) - return loader +def deactivate(): + if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? + sys.path_hooks.remove(ROSImportFinder) + else: # older (trusty) version + sys.path_hooks.remove(_ros_finder_instance_obsolete_python) -sys.meta_path += [ROSFinder()] + sys.path.remove(ROSImportFinder.PATH_TRIGGER) diff --git a/pyros_msgs/importer/tests/srv/TestSrv.srv b/pyros_msgs/importer/tests/srv/TestSrv.srv new file mode 100644 index 0000000..0ae2ee7 --- /dev/null +++ b/pyros_msgs/importer/tests/srv/TestSrv.srv @@ -0,0 +1,4 @@ +string test_request +--- +bool test_response +# TOOD : add more fields for more testing \ No newline at end of file diff --git a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py b/pyros_msgs/importer/tests/test_rosmsg_generator_import.py index 93b6334..48e6213 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_generator_import.py @@ -1,5 +1,7 @@ from __future__ import absolute_import, division, print_function +import importlib + """ Testing executing rosmsg_generator directly (like setup.py would) """ @@ -9,28 +11,95 @@ # Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path -# Importing generator module -from pyros_msgs.importer import rosmsg_generator +# Importing importer module +from pyros_msgs.importer import rosmsg_importer + + +def setup_module(): + rosmsg_importer.activate() -def test_generate_msgsrv_nspkg_usable(): - # generating message class - sitedir, generated_msg, generated_srv = rosmsg_generator.generate_msgsrv_nspkg( - [os.path.join(os.path.dirname(__file__), 'msg', 'TestMsg.msg')], - package='test_gen_msgs', - ns_pkg=True, - ) +def teardown_module(): + rosmsg_importer.deactivate() +# importlib +# https://pymotw.com/3/importlib/index.html +# https://pymotw.com/2/importlib/index.html + +def test_importlib_msg_module_absolute(): # Verify that files exists and are importable - for m in [generated_msg, generated_srv]: - # modules are generated where the file is launched - gen_file = os.path.join(sitedir, *m.split(".")) - assert os.path.exists(gen_file + '.py') or os.path.exists(os.path.join(gen_file, '__init__.py')) + if hasattr(importlib, 'find_loader'): # recent version of import lib + + pkg_loader = importlib.find_loader('test_gen_msgs') + pkg = pkg_loader.load_module() + + subpkg_loader = importlib.find_loader('msg', pkg.__path__) + subpkg = subpkg_loader.load_module() + + loader = importlib.find_loader('TestMsg', subpkg.__path__) + msg_mod = subpkg_loader.load_module() + + else: # old version + msg_mod = importlib.import_module('pyros_msgs.importer.tests.msg.TestMsg') + + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_mod) + else: + pass + + + +# def test_importlib_msg_module_relative(): +# # Verify that files exists and are importable +# # TODO +# # if hasattr(importlib, 'find_loader'): # recent version of import lib +# # +# # pkg_loader = importlib.find_loader('test_gen_msgs') +# # pkg = pkg_loader.load_module() +# # +# # subpkg_loader = importlib.find_loader('msg', pkg.__path__) +# # subpkg = subpkg_loader.load_module() +# # +# # loader = importlib.find_loader('TestMsg', subpkg.__path__) +# # m = subpkg_loader.load_module() +# # +# # else: # old version +# msg_mod = importlib.import_module('.msg.TestMsg', package=__package__) + + + + +def test_importlib_srv_module(): + pass + # TODO + # # Verify that files exists and are importable + # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') + + +# imp https://pymotw.com/2/imp/index.html +# TODO - msg_mod, srv_mod = rosmsg_generator.import_msgsrv(sitedir, generated_msg, generated_srv) +# def test_imp_msg_module(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.msg.TestMsg') +# +# +# def test_imp_msg_pkg(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.msg') +# +# +# def test_imp_srv_module(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.srv.TestSrv') +# +# +# def test_imp_srv_pkg(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.srv') - assert msg_mod is not None - assert srv_mod is not None +# TODO : different python versions... From 18729f7d85750750792cb0530516edc641d6fa5c Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 16 Jun 2017 14:00:56 +0900 Subject: [PATCH 02/28] WIP improving ROS import finder for latest python 2.7 --- pyros_msgs/importer/rosmsg_importer.py | 180 +++++++++--------- .../tests/test_rosmsg_generator_exec.py | 36 ++++ .../tests/test_rosmsg_generator_import.py | 20 ++ 3 files changed, 142 insertions(+), 94 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_importer.py b/pyros_msgs/importer/rosmsg_importer.py index fd801c3..69943ba 100644 --- a/pyros_msgs/importer/rosmsg_importer.py +++ b/pyros_msgs/importer/rosmsg_importer.py @@ -4,6 +4,8 @@ import py +from pyros_msgs.importer import rosmsg_generator + """ A module to setup custom importer for .msg and .srv files Upon import, it will first find the .msg file, then generate the python module for it, then load it. @@ -27,9 +29,13 @@ class ROSLoader(object): - def __init__(self, generator, ext): - self.ext = ext - self.generator = generator + + def __init__(self, path_entry): + self.path_entry = path_entry + + # def __init__(self, generator, ext): + # self.ext = ext + # self.generator = generator # defining this to benefit from backward compat import mechanism in python 3.X def get_filename(self, name): @@ -44,21 +50,6 @@ def load_module(self, fullname): return -# https://pymotw.com/2/sys/imports.html#sys-imports -class NoisyImportFinder(object): - PATH_TRIGGER = 'NoisyImportFinder_PATH_TRIGGER' - - def __init__(self, path_entry): - print('Checking NoisyImportFinder support for %s' % path_entry) - if path_entry != self.PATH_TRIGGER: - print('NoisyImportFinder does not work for %s' % path_entry) - raise ImportError() - return - - def find_module(self, fullname, path=None): - print('NoisyImportFinder looking for "%s"' % fullname) - return None - # https://pymotw.com/3/sys/imports.html#sys-imports # class NoisyImportFinder: @@ -107,95 +98,96 @@ def find_module(self, fullname, path=None): print('ROSImportFinder looking for "%s"' % fullname) return None -elif sys.version_info >= (2, 7, 12): - class ROSImportFinder(object): - PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' - - def __init__(self, path_entry): - self.logger = logging.getLogger(__name__) - self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) - if path_entry != self.PATH_TRIGGER: - self.logger.debug('ROSImportFinder does not work for %s' % path_entry) - raise ImportError() - - self.loaders = { - '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), - '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') - } - - def find_module(self, fullname, path=None): - print('ROSImportFinder looking for "%s"' % fullname) - return None +# elif sys.version_info >= (2, 7, 12): +# class ROSImportFinder(object): +# PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' +# +# def __init__(self, path_entry): +# self.logger = logging.getLogger(__name__) +# self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) +# if path_entry != self.PATH_TRIGGER: +# self.logger.debug('ROSImportFinder does not work for %s' % path_entry) +# raise ImportError() +# +# self.loaders = { +# '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), +# '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') +# } +# +# def find_module(self, fullname, path=None): +# print('ROSImportFinder looking for "%s"' % fullname) +# return None else: class ROSImportFinder(object): + """Find ROS message/service modules""" + def __init__(self, path_entry=None): + # First we need to skip all the cases that we are not concerned with - PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' + if path_entry is None: # called on obsolete python (< 2.7.12) + pass + # else path_entry contains the path where the finder has been instanciated... + elif not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists(os.path.join(path_entry, 'srv')): + raise ImportError # we raise if we cannot find msg or srv folder - def __init__(self, path_entry=None): + # Then we can do the initialisation self.logger = logging.getLogger(__name__) self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) - if path_entry is None: # called on very old python (< 2.7.12) - pass - elif path_entry != self.PATH_TRIGGER: # python following correct PEP ( 302 ?) - self.logger.debug('ROSImportFinder does not work for %s' % path_entry) - raise ImportError() - self.loaders = { - '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), - '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') - } + self.path_entry = path_entry + + def find_module(self, name, path=None): self.logger.debug('ROSImportFinder looking for "%s"' % name) + # on 2.7 + path = path or self.path_entry + + # implementation inspired from pytest.rewrite names = name.rsplit(".", 1) lastname = names[-1] - pth = None - if path is not None: - # Starting with Python 3.3, path is a _NamespacePath(), which - # causes problems if not converted to list. - path = list(path) - if len(path) == 1: - pth = path[0] - - - if pth is None: - - - try: - fd, fn, desc = imp.find_module(lastname, path) - except ImportError: - return None - if fd is not None: - fd.close() - - - - - tp = desc[2] - if tp == imp.PY_COMPILED: - if hasattr(imp, "source_from_cache"): - try: - fn = imp.source_from_cache(fn) - except ValueError: - # Python 3 doesn't like orphaned but still-importable - # .pyc files. - fn = fn[:-1] - else: - fn = fn[:-1] - elif tp != imp.PY_SOURCE: - # Don't know what this is. - return None - else: - fn = os.path.join(pth, name.rpartition(".")[2] + ".py") - - # fn_pypath = py.path.local(fn) - # if not self._should_rewrite(name, fn_pypath, state): - # return None - # - # self._rewritten_names.add(name) + + + if not os.path.exists(os.path.join(path, lastname)): + raise ImportError + elif os.path.isdir(os.path.join(path, lastname)): + + rosf = [f for f in os.listdir(os.path.join(path, lastname)) if os.path.splitext(f)[-1] in ['.msg', '.srv']] + + if rosf: + return ROSLoader(path_entry=os.path.join(path, lastname)) + + # # package case + # for root, dirs, files in os.walk(path, topdown=True): + # + # #rosmsg_generator.genmsgsrv_py(msgsrv_files=[f for f in files if ], package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) + # + # # generated_msg = genmsg_py(msg_files=[f for f in files if f.endswith('.msg')], + # # package=package, + # # outdir_pkg=outdir_pkg, + # # includepath=includepath, + # # initpy=True) # we always create an __init__.py when called from here. + # # generated_srv = gensrv_py(srv_files=[f for f in files if f.endswith('.srv')], + # # package=package, + # # outdir_pkg=outdir_pkg, + # # includepath=includepath, + # # initpy=True) # we always create an __init__.py when called from here. + # + # for name in dirs: + # print(os.path.join(root, name)) + # # rosmsg_generator.genmsgsrv_py(msgsrv_files=msgsrvfiles, package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) + + elif os.path.isfile(os.path.join(path, lastname)): + # module case + pass + + return None + + + + # def find_module(self, name, path=None): # """ @@ -328,7 +320,7 @@ def activate(): for hook in sys.path_hooks: print('Path hook: {}'.format(hook)) - sys.path.insert(0, ROSImportFinder.PATH_TRIGGER) + #sys.path.insert(0, ROSImportFinder.PATH_TRIGGER) def deactivate(): @@ -337,4 +329,4 @@ def deactivate(): else: # older (trusty) version sys.path_hooks.remove(_ros_finder_instance_obsolete_python) - sys.path.remove(ROSImportFinder.PATH_TRIGGER) + #sys.path.remove(ROSImportFinder.PATH_TRIGGER) diff --git a/pyros_msgs/importer/tests/test_rosmsg_generator_exec.py b/pyros_msgs/importer/tests/test_rosmsg_generator_exec.py index c616b95..b5d109c 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_generator_exec.py +++ b/pyros_msgs/importer/tests/test_rosmsg_generator_exec.py @@ -33,3 +33,39 @@ def test_generate_msgsrv_nspkg_usable(): assert msg_mod is not None assert srv_mod is not None +# def test_generate_msgsrv_samepkg_usable(): +# # generating message class +# sitedir, generated_msg, generated_srv = rosmsg_generator.generate_msgsrv_nspkg( +# [os.path.join(os.path.dirname(__file__), 'msg', 'TestMsg.msg')], +# package='test_gen_msgs', +# ns_pkg=True, +# ) +# +# # Verify that files exists and are importable +# for m in [generated_msg, generated_srv]: +# # modules are generated where the file is launched +# gen_file = os.path.join(sitedir, *m.split(".")) +# assert os.path.exists(gen_file + '.py') or os.path.exists(os.path.join(gen_file, '__init__.py')) +# +# msg_mod, srv_mod = rosmsg_generator.import_msgsrv(sitedir, generated_msg, generated_srv) +# +# assert msg_mod is not None +# assert srv_mod is not None + +def test_generate_msgsrv_genpkg_usable(): + # generating message class + sitedir, generated_msg, generated_srv = rosmsg_generator.generate_msgsrv_nspkg( + [os.path.join(os.path.dirname(__file__), 'msg', 'TestMsg.msg')], + package='test_gen_msgs', + ) + + # Verify that files exists and are importable + for m in [generated_msg, generated_srv]: + # modules are generated where the file is launched + gen_file = os.path.join(sitedir, *m.split(".")) + assert os.path.exists(gen_file + '.py') or os.path.exists(os.path.join(gen_file, '__init__.py')) + + msg_mod, srv_mod = rosmsg_generator.import_msgsrv(sitedir, generated_msg, generated_srv) + + assert msg_mod is not None + assert srv_mod is not None \ No newline at end of file diff --git a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py b/pyros_msgs/importer/tests/test_rosmsg_generator_import.py index 48e6213..70f3493 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_generator_import.py @@ -8,6 +8,26 @@ import os import runpy +import logging.config + +logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'default': { + 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + } + }, + 'root': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, +}) # Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path From 06390f86aa092d2b4423d486d510d97a19cc3dc7 Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 16 Jun 2017 16:06:53 +0900 Subject: [PATCH 03/28] WIP importer improvements --- pyros_msgs/importer/rosmsg_importer.py | 162 +++++++++++++++++++++---- 1 file changed, 138 insertions(+), 24 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_importer.py b/pyros_msgs/importer/rosmsg_importer.py index 69943ba..91ca511 100644 --- a/pyros_msgs/importer/rosmsg_importer.py +++ b/pyros_msgs/importer/rosmsg_importer.py @@ -28,27 +28,128 @@ import logging +def _mk_init_name(fullname): + """Return the name of the __init__ module for a given package name.""" + if fullname.endswith('.__init__'): + return fullname + return fullname + '.__init__' + + +def _get_key_name(fullname, db): + """Look in an open shelf for fullname or fullname.__init__, return the name found.""" + if fullname in db: + return fullname + init_name = _mk_init_name(fullname) + if init_name in db: + return init_name + return None + + class ROSLoader(object): - def __init__(self, path_entry): + def __init__(self, path_entry, msgsrv_files, package, outdir_pkg, includepath = None, ns_pkg = None): + + self.logger = logging.getLogger(__name__) self.path_entry = path_entry - # def __init__(self, generator, ext): - # self.ext = ext - # self.generator = generator + self.msgsrv_files = msgsrv_files + self.package = package + self.outdir_pkg = outdir_pkg + + # TODO : we need to determine that from the loader + self.includepath = includepath + self.ns_pkg = ns_pkg + + + def _generate(self, fullname): + + gen = rosmsg_generator.genmsgsrv_py( + msgsrv_files=self.msgsrv_files, + package=self.package, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + ns_pkg=self.ns_pkg + ) + return gen + + def get_source(self, fullname): + self.logger.debug('loading source for "%s" from msg' % fullname) + try: + + self._generate() + + # with shelve_context(self.path_entry) as db: + # key_name = _get_key_name(fullname, db) + # if key_name: + # return db[key_name] + # raise ImportError('could not find source for %s' % fullname) + + pass + + + except Exception as e: + self.logger.debug('could not load source:', e) + raise ImportError(str(e)) + # defining this to benefit from backward compat import mechanism in python 3.X def get_filename(self, name): os.sep.join(name.split(".")) + '.' + self.ext + def _get_filename(self, fullname): + # Make up a fake filename that starts with the path entry + # so pkgutil.get_data() works correctly. + return os.path.join(self.path_entry, fullname) + # defining this to benefit from backward compat import mechanism in python 3.X def is_package(self, name): + names = name.split(".") + parent_idx = len(names) -1 + # trying to find a parent already loaded + while 0<= parent_idx < len(names) : + if names[parent_idx] in sys.modules: # we found a parent, we can get his path and go back + pass + else: # parent not found, need to check its parent + parent_idx-=1 + + + else: # No loaded parent found, lets attempt to import it directly (searching in all sys.paths) + + pass + + return None # TODO : implement check def load_module(self, fullname): - - return + if fullname in sys.modules: + self.logger.debug('reusing existing module from previous import of "%s"' % fullname) + mod = sys.modules[fullname] + else: + self.logger.debug('creating a new module object for "%s"' % fullname) + mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + + # Set a few properties required by PEP 302 + mod.__file__ = self._get_filename(fullname) + mod.__name__ = fullname + mod.__path__ = self.path_entry + mod.__loader__ = self + mod.__package__ = '.'.join(fullname.split('.')[:-1]) + + if self.is_package(fullname): + self.logger.debug('adding path for package') + # Set __path__ for packages + # so we can find the sub-modules. + mod.__path__ = [self.path_entry] + else: + self.logger.debug('imported as regular module') + + source = self.get_source(fullname) + + self.logger.debug('execing source...') + exec source in mod.__dict__ + self.logger.debug('done') + return mod # https://pymotw.com/3/sys/imports.html#sys-imports @@ -78,7 +179,7 @@ def load_module(self, fullname): if sys.version_info >= (3, ): class ROSImportFinder(object): - + # TODO : we can use this to enable ROS Finder only on relevant paths (ros distro paths + dev workspaces) from sys.path PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' def __init__(self, path_entry): @@ -124,10 +225,8 @@ class ROSImportFinder(object): def __init__(self, path_entry=None): # First we need to skip all the cases that we are not concerned with - if path_entry is None: # called on obsolete python (< 2.7.12) - pass - # else path_entry contains the path where the finder has been instanciated... - elif not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists(os.path.join(path_entry, 'srv')): + # path_entry contains the path where the finder has been instantiated... + if not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists(os.path.join(path_entry, 'srv')): raise ImportError # we raise if we cannot find msg or srv folder # Then we can do the initialisation @@ -137,14 +236,13 @@ def __init__(self, path_entry=None): self.path_entry = path_entry - + # This is called for the first unknown (not loaded) name inside path def find_module(self, name, path=None): self.logger.debug('ROSImportFinder looking for "%s"' % name) # on 2.7 path = path or self.path_entry - # implementation inspired from pytest.rewrite names = name.rsplit(".", 1) lastname = names[-1] @@ -157,7 +255,14 @@ def find_module(self, name, path=None): rosf = [f for f in os.listdir(os.path.join(path, lastname)) if os.path.splitext(f)[-1] in ['.msg', '.srv']] if rosf: - return ROSLoader(path_entry=os.path.join(path, lastname)) + return ROSLoader( + msgsrv_files=rosf, + path_entry=os.path.join(path, lastname), + package=name, + outdir_pkg=path, # TODO: if we cannot write into source, use tempfile + #includepath=, + #ns_pkg=, + ) # # package case # for root, dirs, files in os.walk(path, topdown=True): @@ -181,7 +286,14 @@ def find_module(self, name, path=None): elif os.path.isfile(os.path.join(path, lastname)): # module case - pass + return ROSLoader( + msgsrv_files=rosf, + path_entry=os.path.join(path, lastname), + package=name, + outdir_pkg=path, # TODO: if we cannot write into source, use tempfile + # includepath=, + # ns_pkg=, + ) return None @@ -308,25 +420,27 @@ def find_module(self, name, path=None): # self.modules[name] = co, pyc # return self -_ros_finder_instance_obsolete_python = ROSImportFinder +#_ros_finder_instance_obsolete_python = ROSImportFinder def activate(): - if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? - sys.path_hooks.append(ROSImportFinder) - else: # older (trusty) version - sys.path_hooks.append(_ros_finder_instance_obsolete_python) + #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? + sys.path_hooks.append(ROSImportFinder) + # else: # older (trusty) version + # sys.path_hooks.append(_ros_finder_instance_obsolete_python) for hook in sys.path_hooks: print('Path hook: {}'.format(hook)) + # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths #sys.path.insert(0, ROSImportFinder.PATH_TRIGGER) def deactivate(): - if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? - sys.path_hooks.remove(ROSImportFinder) - else: # older (trusty) version - sys.path_hooks.remove(_ros_finder_instance_obsolete_python) + #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? + sys.path_hooks.remove(ROSImportFinder) + # else: # older (trusty) version + # sys.path_hooks.remove(_ros_finder_instance_obsolete_python) + # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths #sys.path.remove(ROSImportFinder.PATH_TRIGGER) From 7a6629c12ba07b5abb5c127ff6561718af4fd246 Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 16 Jun 2017 18:20:14 +0900 Subject: [PATCH 04/28] WIP improving ROS import finder for python 3.5 --- pyros_msgs/importer/rosmsg_importer.py | 40 +++++-- .../tests/test_rosmsg_generator_import.py | 110 +++++++++++++++--- 2 files changed, 125 insertions(+), 25 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_importer.py b/pyros_msgs/importer/rosmsg_importer.py index 91ca511..44734a6 100644 --- a/pyros_msgs/importer/rosmsg_importer.py +++ b/pyros_msgs/importer/rosmsg_importer.py @@ -147,7 +147,7 @@ def load_module(self, fullname): source = self.get_source(fullname) self.logger.debug('execing source...') - exec source in mod.__dict__ + exec(source, mod.__dict__) self.logger.debug('done') return mod @@ -177,23 +177,39 @@ def load_module(self, fullname): -if sys.version_info >= (3, ): - class ROSImportFinder(object): +if sys.version_info >= (3, 5): + + import importlib.abc + + class ROSImportFinder(importlib.abc.PathEntryFinder): # TODO : we can use this to enable ROS Finder only on relevant paths (ros distro paths + dev workspaces) from sys.path PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' def __init__(self, path_entry): + super(ROSImportFinder, self).__init__() + + # First we need to skip all the cases that we are not concerned with + + # if path_entry != self.PATH_TRIGGER: + # self.logger.debug('ROSImportFinder does not work for %s' % path_entry) + # raise ImportError() + + # path_entry contains the path where the finder has been instantiated... + if not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists( + os.path.join(path_entry, 'srv')): + raise ImportError # we raise if we cannot find msg or srv folder + + # Then we can do the initialisation self.logger = logging.getLogger(__name__) self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) - if path_entry != self.PATH_TRIGGER: - self.logger.debug('ROSImportFinder does not work for %s' % path_entry) - raise ImportError() - self.loaders = { - '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), - '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') - } + self.path_entry = path_entry + + + def find_spec(self, fullname, target = None): + print('ROSImportFinder looking for "%s"' % fullname) + return None def find_module(self, fullname, path=None): print('ROSImportFinder looking for "%s"' % fullname) @@ -433,7 +449,7 @@ def activate(): print('Path hook: {}'.format(hook)) # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths - #sys.path.insert(0, ROSImportFinder.PATH_TRIGGER) + # sys.path.insert(1, ROSImportFinder.PATH_TRIGGER) def deactivate(): @@ -443,4 +459,4 @@ def deactivate(): # sys.path_hooks.remove(_ros_finder_instance_obsolete_python) # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths - #sys.path.remove(ROSImportFinder.PATH_TRIGGER) + # sys.path.remove(ROSImportFinder.PATH_TRIGGER) diff --git a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py b/pyros_msgs/importer/tests/test_rosmsg_generator_import.py index 70f3493..ec1754b 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_generator_import.py @@ -1,12 +1,11 @@ from __future__ import absolute_import, division, print_function -import importlib - """ Testing executing rosmsg_generator directly (like setup.py would) """ import os +import sys import runpy import logging.config @@ -31,6 +30,8 @@ # Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path +import importlib + # Importing importer module from pyros_msgs.importer import rosmsg_importer @@ -47,21 +48,104 @@ def teardown_module(): # https://pymotw.com/3/importlib/index.html # https://pymotw.com/2/importlib/index.html -def test_importlib_msg_module_absolute(): +if sys.version_info > (3, 4): + import importlib.util + # REF : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module + # Useful to display details when debugging... + def import_module(name, package=None): + """An approximate implementation of import.""" + absolute_name = importlib.util.resolve_name(name, package) + try: + return sys.modules[absolute_name] + except KeyError: + pass + + path = None + if '.' in absolute_name: + parent_name, _, child_name = absolute_name.rpartition('.') + parent_module = import_module(parent_name) + path = parent_module.spec.submodule_search_locations # this currently breaks (probably because of pytest custom importer ? ) + for finder in sys.meta_path: + spec = finder.find_spec(absolute_name, path) + if spec is not None: + break + else: + raise ImportError('No module named {absolute_name!r}'.format(**locals())) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[absolute_name] = module + if path is not None: + setattr(parent_module, child_name, module) + return module + + +if sys.version_info > (3, 4): + import importlib.util + + # Adapted from : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module + # Useful to debug details... + def import_module(name, package=None): + # using find_spec to use our finder + spec = importlib.util.find_spec(name, package) + + # path = None + # if '.' in absolute_name: + # parent_name, _, child_name = absolute_name.rpartition('.') + # # recursive import call + # parent_module = import_module(parent_name) + # # getting the path instead of relying on spec (not managed by pytest it seems...) + # path = [os.path.join(p, child_name) for p in parent_module.__path__ if os.path.exists(os.path.join(p, child_name))] + + # getting spec with importlib.util (instead of meta path finder to avoid uncompatible pytest finder) + #spec = importlib.util.spec_from_file_location(absolute_name, path[0]) + parent_module = None + child_name = None + if spec is None: + # We didnt find anything, but this is expected on ros packages that haven't been generated yet + if '.' in name: + parent_name, _, child_name = name.rpartition('.') + # recursive import call + parent_module = import_module(parent_name) + # we can check if there is a module in there.. + path = [os.path.join(p, child_name) + for p in parent_module.__path__._path + if os.path.exists(os.path.join(p, child_name))] + # we attempt to get the spec from the first found location + while path and spec is None: + spec = importlib.util.spec_from_file_location(name, path[0]) + path[:] = path[1:] + else: + raise ImportError + + # checking again in case spec has been modified + if spec is not None: + if spec.name not in sys.modules: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[name] = module + if parent_module is not None and child_name is not None: + setattr(parent_module, child_name, sys.modules[name]) + return sys.modules[name] + else: + raise ImportError +else: + def import_module(name, package=None): + return importlib.import_module(name=name, package=package) + + +def test_import_msg_module_absolute(): # Verify that files exists and are importable - if hasattr(importlib, 'find_loader'): # recent version of import lib - - pkg_loader = importlib.find_loader('test_gen_msgs') - pkg = pkg_loader.load_module() + if hasattr(importlib, '__import__'): + msg_mod = importlib.__import__('pyros_msgs.importer.tests.msg.TestMsg',) + else: + import pyros_msgs.importer.tests.msg.TestMsg as msg_mod - subpkg_loader = importlib.find_loader('msg', pkg.__path__) - subpkg = subpkg_loader.load_module() + assert msg_mod is not None - loader = importlib.find_loader('TestMsg', subpkg.__path__) - msg_mod = subpkg_loader.load_module() - else: # old version - msg_mod = importlib.import_module('pyros_msgs.importer.tests.msg.TestMsg') +def test_importlib_msg_module_absolute(): + # Verify that files exists and are dynamically importable + msg_mod = import_module('pyros_msgs.importer.tests.msg.TestMsg') if hasattr(importlib, 'reload'): # recent version of importlib # attempting to reload From 72d2a0b7414a2cee3382bbc6a867fe4c9fcccb17 Mon Sep 17 00:00:00 2001 From: alexv Date: Mon, 19 Jun 2017 17:30:43 +0900 Subject: [PATCH 05/28] WIP improved test structure, now using more basic unittest. --- pyros_msgs/importer/rosmsg_importer.py | 19 +- .../tests/test_rosmsg_generator_import.py | 341 ++++++++++++------ 2 files changed, 244 insertions(+), 116 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_importer.py b/pyros_msgs/importer/rosmsg_importer.py index 44734a6..b70f4ba 100644 --- a/pyros_msgs/importer/rosmsg_importer.py +++ b/pyros_msgs/importer/rosmsg_importer.py @@ -181,9 +181,8 @@ def load_module(self, fullname): import importlib.abc - class ROSImportFinder(importlib.abc.PathEntryFinder): + class ROSImportFinder(object): #importlib.abc.PathEntryFinder): # TODO : we can use this to enable ROS Finder only on relevant paths (ros distro paths + dev workspaces) from sys.path - PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' def __init__(self, path_entry): @@ -196,8 +195,7 @@ def __init__(self, path_entry): # raise ImportError() # path_entry contains the path where the finder has been instantiated... - if not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists( - os.path.join(path_entry, 'srv')): + if not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists(os.path.join(path_entry, 'srv')): raise ImportError # we raise if we cannot find msg or srv folder # Then we can do the initialisation @@ -215,6 +213,7 @@ def find_module(self, fullname, path=None): print('ROSImportFinder looking for "%s"' % fullname) return None + # elif sys.version_info >= (2, 7, 12): # class ROSImportFinder(object): # PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' @@ -303,8 +302,8 @@ def find_module(self, name, path=None): elif os.path.isfile(os.path.join(path, lastname)): # module case return ROSLoader( - msgsrv_files=rosf, - path_entry=os.path.join(path, lastname), + msgsrv_files=os.path.join(path, lastname), + path_entry=path, package=name, outdir_pkg=path, # TODO: if we cannot write into source, use tempfile # includepath=, @@ -442,15 +441,13 @@ def find_module(self, name, path=None): def activate(): #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? sys.path_hooks.append(ROSImportFinder) + # else: # older (trusty) version # sys.path_hooks.append(_ros_finder_instance_obsolete_python) for hook in sys.path_hooks: print('Path hook: {}'.format(hook)) - # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths - # sys.path.insert(1, ROSImportFinder.PATH_TRIGGER) - def deactivate(): #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? @@ -458,5 +455,5 @@ def deactivate(): # else: # older (trusty) version # sys.path_hooks.remove(_ros_finder_instance_obsolete_python) - # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths - # sys.path.remove(ROSImportFinder.PATH_TRIGGER) + +# TODO : a meta finder could find a full ROS distro... diff --git a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py b/pyros_msgs/importer/tests/test_rosmsg_generator_import.py index ec1754b..421e594 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_generator_import.py @@ -1,5 +1,4 @@ from __future__ import absolute_import, division, print_function - """ Testing executing rosmsg_generator directly (like setup.py would) """ @@ -28,6 +27,9 @@ }, }) +# Since test frameworks (like pytest) play with the import machinery, we cannot use it here... +import unittest + # Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path import importlib @@ -35,123 +37,250 @@ # Importing importer module from pyros_msgs.importer import rosmsg_importer +# importlib +# https://pymotw.com/3/importlib/index.html +# https://pymotw.com/2/importlib/index.html -def setup_module(): - rosmsg_importer.activate() +# if sys.version_info > (3, 4): +# import importlib.util +# # REF : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module +# # Useful to display details when debugging... +# def import_module(name, package=None): +# """An approximate implementation of import.""" +# absolute_name = importlib.util.resolve_name(name, package) +# try: +# return sys.modules[absolute_name] +# except KeyError: +# pass +# +# path = None +# if '.' in absolute_name: +# parent_name, _, child_name = absolute_name.rpartition('.') +# parent_module = import_module(parent_name) +# path = parent_module.spec.submodule_search_locations # this currently breaks (probably because of pytest custom importer ? ) +# for finder in sys.meta_path: +# spec = finder.find_spec(absolute_name, path) +# if spec is not None: +# break +# else: +# raise ImportError('No module named {absolute_name!r}'.format(**locals())) +# module = importlib.util.module_from_spec(spec) +# spec.loader.exec_module(module) +# sys.modules[absolute_name] = module +# if path is not None: +# setattr(parent_module, child_name, module) +# return module + + +# if sys.version_info > (3, 4): +# import importlib.util +# +# # Adapted from : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module +# # Useful to debug details... +# def import_module(name, package=None): +# # using find_spec to use our finder +# spec = importlib.util.find_spec(name, package) +# +# # path = None +# # if '.' in absolute_name: +# # parent_name, _, child_name = absolute_name.rpartition('.') +# # # recursive import call +# # parent_module = import_module(parent_name) +# # # getting the path instead of relying on spec (not managed by pytest it seems...) +# # path = [os.path.join(p, child_name) for p in parent_module.__path__ if os.path.exists(os.path.join(p, child_name))] +# +# # getting spec with importlib.util (instead of meta path finder to avoid uncompatible pytest finder) +# #spec = importlib.util.spec_from_file_location(absolute_name, path[0]) +# parent_module = None +# child_name = None +# if spec is None: +# # We didnt find anything, but this is expected on ros packages that haven't been generated yet +# if '.' in name: +# parent_name, _, child_name = name.rpartition('.') +# # recursive import call +# parent_module = import_module(parent_name) +# # we can check if there is a module in there.. +# path = [os.path.join(p, child_name) +# for p in parent_module.__path__._path +# if os.path.exists(os.path.join(p, child_name))] +# # we attempt to get the spec from the first found location +# while path and spec is None: +# spec = importlib.util.spec_from_file_location(name, path[0]) +# path[:] = path[1:] +# else: +# raise ImportError +# +# # checking again in case spec has been modified +# if spec is not None: +# if spec.name not in sys.modules: +# module = importlib.util.module_from_spec(spec) +# spec.loader.exec_module(module) +# sys.modules[name] = module +# if parent_module is not None and child_name is not None: +# setattr(parent_module, child_name, sys.modules[name]) +# return sys.modules[name] +# else: +# raise ImportError +# else: +# def import_module(name, package=None): +# return importlib.import_module(name=name, package=package) -def teardown_module(): - rosmsg_importer.deactivate() +# +# Note : we cannot assume anything about import implementation (different python version, different version of pytest) +# => we need to test them all... +# +# HACK to fix spec from pytest +# @property +# def spec(): +# importlib.util.spec_from_file_location(__file__) + + +def print_importers(): + import sys + import pprint + + print('PATH:'), + pprint.pprint(sys.path) + print() + print('IMPORTERS:') + for name, cache_value in sys.path_importer_cache.items(): + name = name.replace(sys.prefix, '...') + print('%s: %r' % (name, cache_value)) + + +class TestImportAnotherMsg(unittest.TestCase): + @classmethod + def setUpClass(cls): + # trying to avoid cache pollution of consecutive tests.. + if hasattr(importlib, 'invalidate_caches'): + importlib.invalidate_caches() + rosmsg_importer.activate() + + @classmethod + def tearDownClass(cls): + rosmsg_importer.deactivate() + + def test_import_absolute(self): + print_importers() + # Verify that files exists and are importable + import std_msgs.msg.Bool as msg_bool + + assert msg_bool is not None + + def test_import_from(self): + print_importers() + # Verify that files exists and are importable + from std_msgs.msg import Bool as msg_bool + + assert msg_bool is not None + + @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + def test_importlib_import_absolute(self): + # Verify that files exists and are importable + msg_bool = importlib.__import__('std_msgs.msg.Bool',) + + assert msg_bool is not None + + @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), + reason="importlib does not have attribute import_module") + def test_importlib_loadmodule_absolute(self): + # Verify that files exists and are dynamically importable + pkg_list = 'std_msgs.msg.Bool'.split('.')[:-1] + mod_list = 'std_msgs.msg.Bool'.split('.')[1:] + pkg = None + for pkg_name, mod_name in zip(pkg_list, mod_list): + pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) + pkg = pkg_loader.load_module(mod_name) + + msg_mod = pkg + + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_mod) + else: + pass -# importlib -# https://pymotw.com/3/importlib/index.html -# https://pymotw.com/2/importlib/index.html + # TODO : dynamic using module_spec (python 3.5) -if sys.version_info > (3, 4): - import importlib.util - # REF : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module - # Useful to display details when debugging... - def import_module(name, package=None): - """An approximate implementation of import.""" - absolute_name = importlib.util.resolve_name(name, package) - try: - return sys.modules[absolute_name] - except KeyError: - pass + @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") + def test_importlib_importmodule_absolute(self): + # Verify that files exists and are dynamically importable + msg_bool = importlib.import_module('std_msgs.msg.Bool') - path = None - if '.' in absolute_name: - parent_name, _, child_name = absolute_name.rpartition('.') - parent_module = import_module(parent_name) - path = parent_module.spec.submodule_search_locations # this currently breaks (probably because of pytest custom importer ? ) - for finder in sys.meta_path: - spec = finder.find_spec(absolute_name, path) - if spec is not None: - break - else: - raise ImportError('No module named {absolute_name!r}'.format(**locals())) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - sys.modules[absolute_name] = module - if path is not None: - setattr(parent_module, child_name, module) - return module - - -if sys.version_info > (3, 4): - import importlib.util - - # Adapted from : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module - # Useful to debug details... - def import_module(name, package=None): - # using find_spec to use our finder - spec = importlib.util.find_spec(name, package) - - # path = None - # if '.' in absolute_name: - # parent_name, _, child_name = absolute_name.rpartition('.') - # # recursive import call - # parent_module = import_module(parent_name) - # # getting the path instead of relying on spec (not managed by pytest it seems...) - # path = [os.path.join(p, child_name) for p in parent_module.__path__ if os.path.exists(os.path.join(p, child_name))] - - # getting spec with importlib.util (instead of meta path finder to avoid uncompatible pytest finder) - #spec = importlib.util.spec_from_file_location(absolute_name, path[0]) - parent_module = None - child_name = None - if spec is None: - # We didnt find anything, but this is expected on ros packages that haven't been generated yet - if '.' in name: - parent_name, _, child_name = name.rpartition('.') - # recursive import call - parent_module = import_module(parent_name) - # we can check if there is a module in there.. - path = [os.path.join(p, child_name) - for p in parent_module.__path__._path - if os.path.exists(os.path.join(p, child_name))] - # we attempt to get the spec from the first found location - while path and spec is None: - spec = importlib.util.spec_from_file_location(name, path[0]) - path[:] = path[1:] - else: - raise ImportError - - # checking again in case spec has been modified - if spec is not None: - if spec.name not in sys.modules: - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - sys.modules[name] = module - if parent_module is not None and child_name is not None: - setattr(parent_module, child_name, sys.modules[name]) - return sys.modules[name] + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_bool) else: - raise ImportError -else: - def import_module(name, package=None): - return importlib.import_module(name=name, package=package) + pass + assert msg_bool is not None -def test_import_msg_module_absolute(): - # Verify that files exists and are importable - if hasattr(importlib, '__import__'): - msg_mod = importlib.__import__('pyros_msgs.importer.tests.msg.TestMsg',) - else: + +class TestImportSameMsg(unittest.TestCase): + @classmethod + def setUpClass(cls): + # trying to avoid cache pollution of consecutive tests.. + if hasattr(importlib, 'invalidate_caches'): + importlib.invalidate_caches() + rosmsg_importer.activate() + + @classmethod + def tearDownClass(cls): + rosmsg_importer.deactivate() + + def test_import_absolute(self): + print_importers() + # Verify that files exists and are importable import pyros_msgs.importer.tests.msg.TestMsg as msg_mod - assert msg_mod is not None + assert msg_mod is not None + + def test_import_relative(self): + print_importers() + # Verify that files exists and are importable + from .msg import TestMsg as msg_mod + + assert msg_mod is not None + + @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + def test_importlib_import_absolute(self): + # Verify that files exists and are importable + msg_mod = importlib.__import__('pyros_msgs.importer.tests.msg.TestMsg',) + + assert msg_mod is not None + @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute import_module") + def test_importlib_loadmodule_absolute(self): + # Verify that files exists and are dynamically importable + pkg_list = 'pyros_msgs.importer.tests.msg.TestMsg'.split('.')[:-1] + mod_list = 'pyros_msgs.importer.tests.msg.TestMsg'.split('.')[1:] + pkg = None + for pkg_name, mod_name in zip(pkg_list, mod_list): + pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) + pkg = pkg_loader.load_module(mod_name) -def test_importlib_msg_module_absolute(): - # Verify that files exists and are dynamically importable - msg_mod = import_module('pyros_msgs.importer.tests.msg.TestMsg') + msg_mod = pkg - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(msg_mod) - else: - pass + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_mod) + else: + pass + + # TODO : dynamic using specs (python 3.5) + + @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") + def test_importlib_importmodule_absolute(self): + # Verify that files exists and are dynamically importable + msg_mod = importlib.import_module('pyros_msgs.importer.tests.msg.TestMsg') + + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_mod) + else: + pass @@ -205,5 +334,7 @@ def test_importlib_srv_module(): # msg_mod = imp.import_module('test_gen_msgs.srv') -# TODO : different python versions... - +if __name__ == '__main__': + unittest.main() + #import nose + #nose.main() From 41a2873ea4f78daec08af6fd63a46e7a572f14a1 Mon Sep 17 00:00:00 2001 From: AlexV Date: Tue, 20 Jun 2017 17:46:49 +0900 Subject: [PATCH 06/28] WIP extracted tests out of package to be able to test importing from existing python package, without having the python package already imported. --- pyros_msgs/importer/rosmsg_finder.py | 531 ++++++++++++++++++ pyros_msgs/importer/rosmsg_importer.py | 459 --------------- pyros_msgs/importer/rosmsg_loader.py | 267 +++++++++ pyros_msgs/importer/rosmsg_metafinder.py | 192 +++++++ pyros_msgs/importer/tests/__init__.py | 24 +- ....py => test_rosmsg_import_pkg_absolute.py} | 115 ++-- .../test_rosmsg_import_msgmod_relative.py | 272 +++++++++ .../test_rosmsg_import_msgpkg_relative.py | 275 +++++++++ 8 files changed, 1598 insertions(+), 537 deletions(-) create mode 100644 pyros_msgs/importer/rosmsg_finder.py delete mode 100644 pyros_msgs/importer/rosmsg_importer.py create mode 100644 pyros_msgs/importer/rosmsg_loader.py create mode 100644 pyros_msgs/importer/rosmsg_metafinder.py rename pyros_msgs/importer/tests/{test_rosmsg_generator_import.py => test_rosmsg_import_pkg_absolute.py} (74%) create mode 100644 tests/importer/test_rosmsg_import_msgmod_relative.py create mode 100644 tests/importer/test_rosmsg_import_msgpkg_relative.py diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py new file mode 100644 index 0000000..43054ed --- /dev/null +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -0,0 +1,531 @@ +from __future__ import absolute_import, division, print_function + +import contextlib +import importlib +import site + +from pyros_msgs.importer import rosmsg_loader + +""" +A module to setup custom importer for .msg and .srv files +Upon import, it will first find the .msg file, then generate the python module for it, then load it. + +TODO... +""" + +# We need to be extra careful with python versions +# Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module + +# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path +# Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) +import os +import sys + +# This will take the ROS distro version if ros has been setup +import genpy.generator +import genpy.generate_initpy + +import logging + + + +if sys.version_info >= (3, 4): + + import importlib.abc + import importlib.machinery + import importlib.util + + + class ROSFileFinder(importlib.machinery.FileFinder): + + def __init__(self, path, *loader_details): + """ + Finder to get ROS specific files and directories (message and services files). + It need to be inserted in sys.path_hooks before FileFinder, since these are Files but not python ones. + + :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) + """ + super(ROSFileFinder, self).__init__(path, *loader_details) + + # First we need to skip all the cases that we are not concerned with + + # if path_entry != self.PATH_TRIGGER: + # self.logger.debug('ROSImportFinder does not work for %s' % path_entry) + # raise ImportError() + + # Adding the share path (where we can find all packages and their messages) + # share_path = os.path.join(workspace_entry, 'share') + # if os.path.exists(share_path): + # self.share_path = share_path + # + # python_libpath = os.path.join(workspace_entry, 'lib', + # 'python' + sys.version_info.major + '.' + sys.version_info.minor) + # if os.path.exists(python_libpath): + # # adding python site directories for the ROS distro (to find python modules as usual with ROS) + # if os.path.exists(os.path.join(python_libpath, 'dist-packages')): + # site.addsitedir(os.path.join(python_libpath, 'dist-packages')) + # if os.path.exists(os.path.join(python_libpath, 'site-packages')): + # site.addsitedir(os.path.join(python_libpath, 'site-packages')) + # # Note : here we want to keep a pure python logic to not pollute the environment (not like with pyros-setup) + # + # # self.rospkgs = [] + # # for root, dirs, files in os.walk(self.share_path, topdown=False): + # # if 'package.xml' in files: # we have found a ros package + # # + # # + # # + # # self.rospkgs.append(root) + # # found_one = True + # # if not found_one: + # # raise ImportError("No ROS package found in {0}".format(workspace_entry)) + # + # else: # this is not a workspace, maybe we are expected to get the package directly from source ? + # found_one = False + # self.src_rospkgs = [] + # for root, dirs, files in os.walk(workspace_entry, topdown=False): + # if 'package.xml' in files: # we have found a ros package + # self.src_rospkgs.append(root) + # found_one = True + # if not found_one: + # raise ImportError("No ROS package found in {0}".format(workspace_entry)) + + # # path_entry contains the path where the finder has been instantiated... + # if (not os.path.exists(path_entry) or # non existent path + # not os.path.basename(path_entry) in ['msg', 'srv'] # path not terminated with msg/srv (msg and srv inside python source) + # ): + # raise ImportError # we raise if we cannot find msg or srv folder + + # Then we can do the initialisation + self.logger = logging.getLogger(__name__) + self.logger.debug('Checking ROSFileFinder support for %s' % path) + + + def find_spec(self, fullname, target=None): + print('ROSImportFinder looking for "%s"' % fullname) + """ + :param fullname: the name of the package we are trying to import + :param target: what we plan to do with it + :return: + """ + # TODO: read PEP 420 :) + last_mile = fullname.split('.')[-1] + + for src_pkg in self.src_rospkgs: + if os.path.basename(src_pkg) == fullname: # we found it + return importlib.machinery.ModuleSpec( + fullname, + None, # TODO loader + origin=src_pkg, + is_package=True + ) + + # If we get here, we need to find the package in a workspace... + + # If we can find it in share/ with messages, we can generate them. + # good or bad idea ? + # Else, we can use the usual python import (since we added them to site) + #return importlib.util..(fullname). + + for root, dirs, files in os.walk(self.workspace_entry, topdown=False): + if last_mile + '.msg' in files: + return importlib.machinery.ModuleSpec( + fullname, + None, # TODO loader + origin=root, + is_package=False + ) + if last_mile + '.srv' in files: + return importlib.machinery.ModuleSpec( + fullname, + None, # TODO loader + origin=root, + is_package=False + ) + + # we couldn't find the module. let someone else find it. + return None + + + class ROSImportFinder(importlib.abc.PathEntryFinder): + # TODO : we can use this to enable ROS Finder only on relevant paths (ros distro paths + dev workspaces) from sys.path + + def __init__(self, workspace_entry): + """ + Finder to get ROS packages (python code and message files). + It need to be inserted in sys.path_hooks before FileFinder, since these are Files but not python + + Note : it is currently not possible (tested on python 3.4) to build a solid finder only for a set of messages that would be part of a python package, + since sub-packages do not follow the same import logic as modules (finder not called since we are already with a known standard python importer) + Therefore this finder focuses on importing a whole ROS package, and messages have to be in a ROS package. + + :param path_entry: the ROS workspace path (no finder should have been instantiated yet) + """ + super(ROSImportFinder, self).__init__() + + # First we need to skip all the cases that we are not concerned with + + # if path_entry != self.PATH_TRIGGER: + # self.logger.debug('ROSImportFinder does not work for %s' % path_entry) + # raise ImportError() + + # Adding the share path (where we can find all packages and their messages) + share_path = os.path.join(workspace_entry, 'share') + if os.path.exists(share_path): + self.share_path = share_path + + python_libpath = os.path.join(workspace_entry, 'lib', + 'python' + sys.version_info.major + '.' + sys.version_info.minor) + if os.path.exists(python_libpath): + # adding python site directories for the ROS distro (to find python modules as usual with ROS) + if os.path.exists(os.path.join(python_libpath, 'dist-packages')): + site.addsitedir(os.path.join(python_libpath, 'dist-packages')) + if os.path.exists(os.path.join(python_libpath, 'site-packages')): + site.addsitedir(os.path.join(python_libpath, 'site-packages')) + # Note : here we want to keep a pure python logic to not pollute the environment (not like with pyros-setup) + + # self.rospkgs = [] + # for root, dirs, files in os.walk(self.share_path, topdown=False): + # if 'package.xml' in files: # we have found a ros package + # + # + # + # self.rospkgs.append(root) + # found_one = True + # if not found_one: + # raise ImportError("No ROS package found in {0}".format(workspace_entry)) + + else: # this is not a workspace, maybe we are expected to get the package directly from source ? + found_one = False + self.src_rospkgs = [] + for root, dirs, files in os.walk(workspace_entry, topdown=False): + if 'package.xml' in files: # we have found a ros package + self.src_rospkgs.append(root) + found_one = True + if not found_one: + raise ImportError("No ROS package found in {0}".format(workspace_entry)) + + + # # path_entry contains the path where the finder has been instantiated... + # if (not os.path.exists(path_entry) or # non existent path + # not os.path.basename(path_entry) in ['msg', 'srv'] # path not terminated with msg/srv (msg and srv inside python source) + # ): + # raise ImportError # we raise if we cannot find msg or srv folder + + # Then we can do the initialisation + self.logger = logging.getLogger(__name__) + self.logger.debug('Checking ROSImportFinder support for %s' % workspace_entry) + + self.workspace_entry = workspace_entry + + + def find_spec(self, fullname, target = None): + print('ROSImportFinder looking for "%s"' % fullname) + """ + :param fullname: the name of the package we are trying to import + :param target: what we plan to do with it + :return: + """ + # TODO: read PEP 420 :) + last_mile = fullname.split('.')[-1] + + for src_pkg in self.src_rospkgs: + if os.path.basename(src_pkg) == fullname: # we found it + return importlib.machinery.ModuleSpec( + fullname, + None, # TODO loader + origin=src_pkg, + is_package=True + ) + + # If we get here, we need to find the package in a workspace... + + # If we can find it in share/ with messages, we can generate them. + # good or bad idea ? + # Else, we can use the usual python import (since we added them to site) + #return importlib.util.. (fullname). + + + + for root, dirs, files in os.walk(self.workspace_entry, topdown=False): + if last_mile + '.msg' in files: + return importlib.machinery.ModuleSpec( + fullname, + None, # TODO loader + origin=root, + is_package=False + ) + if last_mile + '.srv' in files: + return importlib.machinery.ModuleSpec( + fullname, + None, # TODO loader + origin=root, + is_package=False + ) + + # we couldn't find the module. let someone else find it. + return None + + +else: + class ROSImportFinder(object): + """Find ROS message/service modules""" + def __init__(self, path_entry=None): + # First we need to skip all the cases that we are not concerned with + + # path_entry contains the path where the finder has been instantiated... + if not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists(os.path.join(path_entry, 'srv')): + raise ImportError # we raise if we cannot find msg or srv folder + + # Then we can do the initialisation + self.logger = logging.getLogger(__name__) + self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) + + self.path_entry = path_entry + + + # This is called for the first unknown (not loaded) name inside path + def find_module(self, name, path=None): + self.logger.debug('ROSImportFinder looking for "%s"' % name) + + # on 2.7 + path = path or self.path_entry + + # implementation inspired from pytest.rewrite + names = name.rsplit(".", 1) + lastname = names[-1] + + + if not os.path.exists(os.path.join(path, lastname)): + raise ImportError + elif os.path.isdir(os.path.join(path, lastname)): + + rosf = [f for f in os.listdir(os.path.join(path, lastname)) if os.path.splitext(f)[-1] in ['.msg', '.srv']] + + if rosf: + return ROSLoader( + msgsrv_files=rosf, + path_entry=os.path.join(path, lastname), + package=name, + outdir_pkg=path, # TODO: if we cannot write into source, use tempfile + #includepath=, + #ns_pkg=, + ) + + # # package case + # for root, dirs, files in os.walk(path, topdown=True): + # + # #rosmsg_generator.genmsgsrv_py(msgsrv_files=[f for f in files if ], package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) + # + # # generated_msg = genmsg_py(msg_files=[f for f in files if f.endswith('.msg')], + # # package=package, + # # outdir_pkg=outdir_pkg, + # # includepath=includepath, + # # initpy=True) # we always create an __init__.py when called from here. + # # generated_srv = gensrv_py(srv_files=[f for f in files if f.endswith('.srv')], + # # package=package, + # # outdir_pkg=outdir_pkg, + # # includepath=includepath, + # # initpy=True) # we always create an __init__.py when called from here. + # + # for name in dirs: + # print(os.path.join(root, name)) + # # rosmsg_generator.genmsgsrv_py(msgsrv_files=msgsrvfiles, package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) + + elif os.path.isfile(os.path.join(path, lastname)): + # module case + return ROSLoader( + msgsrv_files=os.path.join(path, lastname), + path_entry=path, + package=name, + outdir_pkg=path, # TODO: if we cannot write into source, use tempfile + # includepath=, + # ns_pkg=, + ) + + return None + + + + + + # def find_module(self, name, path=None): + # """ + # Return the loader for the specified module. + # """ + # # Ref : https://www.python.org/dev/peps/pep-0302/#specification-part-1-the-importer-protocol + # + # # + # loader = None + # + # # path = path or sys.path + # # for p in path: + # # for f in os.listdir(p): + # # filename, ext = os.path.splitext(f) + # # # our modules generated from messages are always a leaf in import tree so we only care about this case + # # if ext in self.loaders.keys() and filename == name.split('.')[-1]: + # # loader = self.loaders.get(ext) + # # break # we found it. break out. + # # + # # return loader + # + # # implementation inspired from pytest.rewrite + # self.logger.debug("find_module called for: %s" % name) + # names = name.rsplit(".", 1) + # lastname = names[-1] + # pth = None + # if path is not None: + # # Starting with Python 3.3, path is a _NamespacePath(), which + # # causes problems if not converted to list. + # path = list(path) + # if len(path) == 1: + # pth = path[0] + # + # + # if pth is None: + # + # + # + # + # + # try: + # fd, fn, desc = imp.find_module(lastname, path) + # except ImportError: + # return None + # if fd is not None: + # fd.close() + # + # + # + # + # tp = desc[2] + # if tp == imp.PY_COMPILED: + # if hasattr(imp, "source_from_cache"): + # try: + # fn = imp.source_from_cache(fn) + # except ValueError: + # # Python 3 doesn't like orphaned but still-importable + # # .pyc files. + # fn = fn[:-1] + # else: + # fn = fn[:-1] + # elif tp != imp.PY_SOURCE: + # # Don't know what this is. + # return None + # else: + # fn = os.path.join(pth, name.rpartition(".")[2] + ".py") + # + # fn_pypath = py.path.local(fn) + # if not self._should_rewrite(name, fn_pypath, state): + # return None + # + # self._rewritten_names.add(name) + # + # # The requested module looks like a test file, so rewrite it. This is + # # the most magical part of the process: load the source, rewrite the + # # asserts, and load the rewritten source. We also cache the rewritten + # # module code in a special pyc. We must be aware of the possibility of + # # concurrent pytest processes rewriting and loading pycs. To avoid + # # tricky race conditions, we maintain the following invariant: The + # # cached pyc is always a complete, valid pyc. Operations on it must be + # # atomic. POSIX's atomic rename comes in handy. + # write = not sys.dont_write_bytecode + # cache_dir = os.path.join(fn_pypath.dirname, "__pycache__") + # if write: + # try: + # os.mkdir(cache_dir) + # except OSError: + # e = sys.exc_info()[1].errno + # if e == errno.EEXIST: + # # Either the __pycache__ directory already exists (the + # # common case) or it's blocked by a non-dir node. In the + # # latter case, we'll ignore it in _write_pyc. + # pass + # elif e in [errno.ENOENT, errno.ENOTDIR]: + # # One of the path components was not a directory, likely + # # because we're in a zip file. + # write = False + # elif e in [errno.EACCES, errno.EROFS, errno.EPERM]: + # state.trace("read only directory: %r" % fn_pypath.dirname) + # write = False + # else: + # raise + # cache_name = fn_pypath.basename[:-3] + PYC_TAIL + # pyc = os.path.join(cache_dir, cache_name) + # # Notice that even if we're in a read-only directory, I'm going + # # to check for a cached pyc. This may not be optimal... + # co = _read_pyc(fn_pypath, pyc, state.trace) + # if co is None: + # state.trace("rewriting %r" % (fn,)) + # source_stat, co = _rewrite_test(self.config, fn_pypath) + # if co is None: + # # Probably a SyntaxError in the test. + # return None + # if write: + # _make_rewritten_pyc(state, source_stat, pyc, co) + # else: + # state.trace("found cached rewritten pyc for %r" % (fn,)) + # self.modules[name] = co, pyc + # return self + + +# Useless ? +#_ros_finder_instance_obsolete_python = ROSImportFinder + +ros_distro_finder = None + + +def activate(rosdistro_path=None, *workspaces): + global ros_distro_finder + if rosdistro_path is None: # autodetect most recent installed distro + if os.path.exists('/opt/ros/lunar'): + rosdistro_path = '/opt/ros/lunar' + elif os.path.exists('/opt/ros/kinetic'): + rosdistro_path = '/opt/ros/kinetic' + elif os.path.exists('/opt/ros/jade'): + rosdistro_path = '/opt/ros/jade' + elif os.path.exists('/opt/ros/indigo'): + rosdistro_path = '/opt/ros/indigo' + else: + raise ImportError( + "No ROS distro detected on this system. Please specify the path when calling ROSImportMetaFinder()") + + ros_distro_finder = ROSImportMetaFinder(rosdistro_path, *workspaces) + sys.meta_path.append(ros_distro_finder) + + #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? + + # We need to be before FileFinder to be able to find our (non .py[c]) files + # inside, maybe already imported, python packages... + sys.path_hooks.insert(1, ROSImportFinder) + + # else: # older (trusty) version + # sys.path_hooks.append(_ros_finder_instance_obsolete_python) + + for hook in sys.path_hooks: + print('Path hook: {}'.format(hook)) + + # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths + if paths: + sys.path.append(*paths) + + +def deactivate(*paths): + """ CAREFUL : even if we remove our path_hooks, the created finder are still cached in sys.path_importer_cache.""" + #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? + sys.path_hooks.remove(ROSImportFinder) + # else: # older (trusty) version + # sys.path_hooks.remove(_ros_finder_instance_obsolete_python) + if paths: + sys.path.remove(*paths) + + sys.meta_path.remove(ros_distro_finder) + + +@contextlib.contextmanager +def ROSImportContext(*paths): + activate(*paths) + yield + deactivate(*paths) + + +# TODO : a meta finder could find a full ROS distro... diff --git a/pyros_msgs/importer/rosmsg_importer.py b/pyros_msgs/importer/rosmsg_importer.py deleted file mode 100644 index b70f4ba..0000000 --- a/pyros_msgs/importer/rosmsg_importer.py +++ /dev/null @@ -1,459 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import imp - -import py - -from pyros_msgs.importer import rosmsg_generator - -""" -A module to setup custom importer for .msg and .srv files -Upon import, it will first find the .msg file, then generate the python module for it, then load it. - -TODO... -""" - -# We need to be extra careful with python versions -# Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path -# Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) -import os -import sys - -# This will take the ROS distro version if ros has been setup -import genpy.generator -import genpy.generate_initpy - -import logging - - -def _mk_init_name(fullname): - """Return the name of the __init__ module for a given package name.""" - if fullname.endswith('.__init__'): - return fullname - return fullname + '.__init__' - - -def _get_key_name(fullname, db): - """Look in an open shelf for fullname or fullname.__init__, return the name found.""" - if fullname in db: - return fullname - init_name = _mk_init_name(fullname) - if init_name in db: - return init_name - return None - - -class ROSLoader(object): - - def __init__(self, path_entry, msgsrv_files, package, outdir_pkg, includepath = None, ns_pkg = None): - - self.logger = logging.getLogger(__name__) - self.path_entry = path_entry - - self.msgsrv_files = msgsrv_files - self.package = package - self.outdir_pkg = outdir_pkg - - # TODO : we need to determine that from the loader - self.includepath = includepath - self.ns_pkg = ns_pkg - - - def _generate(self, fullname): - - gen = rosmsg_generator.genmsgsrv_py( - msgsrv_files=self.msgsrv_files, - package=self.package, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - ns_pkg=self.ns_pkg - ) - return gen - - def get_source(self, fullname): - self.logger.debug('loading source for "%s" from msg' % fullname) - try: - - self._generate() - - # with shelve_context(self.path_entry) as db: - # key_name = _get_key_name(fullname, db) - # if key_name: - # return db[key_name] - # raise ImportError('could not find source for %s' % fullname) - - pass - - - except Exception as e: - self.logger.debug('could not load source:', e) - raise ImportError(str(e)) - - - # defining this to benefit from backward compat import mechanism in python 3.X - def get_filename(self, name): - os.sep.join(name.split(".")) + '.' + self.ext - - def _get_filename(self, fullname): - # Make up a fake filename that starts with the path entry - # so pkgutil.get_data() works correctly. - return os.path.join(self.path_entry, fullname) - - # defining this to benefit from backward compat import mechanism in python 3.X - def is_package(self, name): - names = name.split(".") - parent_idx = len(names) -1 - # trying to find a parent already loaded - while 0<= parent_idx < len(names) : - if names[parent_idx] in sys.modules: # we found a parent, we can get his path and go back - pass - else: # parent not found, need to check its parent - parent_idx-=1 - - - else: # No loaded parent found, lets attempt to import it directly (searching in all sys.paths) - - pass - - - return None # TODO : implement check - - def load_module(self, fullname): - - if fullname in sys.modules: - self.logger.debug('reusing existing module from previous import of "%s"' % fullname) - mod = sys.modules[fullname] - else: - self.logger.debug('creating a new module object for "%s"' % fullname) - mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) - - # Set a few properties required by PEP 302 - mod.__file__ = self._get_filename(fullname) - mod.__name__ = fullname - mod.__path__ = self.path_entry - mod.__loader__ = self - mod.__package__ = '.'.join(fullname.split('.')[:-1]) - - if self.is_package(fullname): - self.logger.debug('adding path for package') - # Set __path__ for packages - # so we can find the sub-modules. - mod.__path__ = [self.path_entry] - else: - self.logger.debug('imported as regular module') - - source = self.get_source(fullname) - - self.logger.debug('execing source...') - exec(source, mod.__dict__) - self.logger.debug('done') - return mod - -# https://pymotw.com/3/sys/imports.html#sys-imports - -# class NoisyImportFinder: -# -# PATH_TRIGGER = 'NoisyImportFinder_PATH_TRIGGER' -# -# def __init__(self, path_entry): -# print('Checking {}:'.format(path_entry), end=' ') -# if path_entry != self.PATH_TRIGGER: -# print('wrong finder') -# raise ImportError() -# else: -# print('works') -# return -# -# def find_module(self, fullname, path=None): -# print('Looking for {!r}'.format(fullname)) -# return None -# -# -# sys.path_hooks.append(NoisyImportFinder) - - - - - -if sys.version_info >= (3, 5): - - import importlib.abc - - class ROSImportFinder(object): #importlib.abc.PathEntryFinder): - # TODO : we can use this to enable ROS Finder only on relevant paths (ros distro paths + dev workspaces) from sys.path - - def __init__(self, path_entry): - - super(ROSImportFinder, self).__init__() - - # First we need to skip all the cases that we are not concerned with - - # if path_entry != self.PATH_TRIGGER: - # self.logger.debug('ROSImportFinder does not work for %s' % path_entry) - # raise ImportError() - - # path_entry contains the path where the finder has been instantiated... - if not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists(os.path.join(path_entry, 'srv')): - raise ImportError # we raise if we cannot find msg or srv folder - - # Then we can do the initialisation - self.logger = logging.getLogger(__name__) - self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) - - self.path_entry = path_entry - - - def find_spec(self, fullname, target = None): - print('ROSImportFinder looking for "%s"' % fullname) - return None - - def find_module(self, fullname, path=None): - print('ROSImportFinder looking for "%s"' % fullname) - return None - - -# elif sys.version_info >= (2, 7, 12): -# class ROSImportFinder(object): -# PATH_TRIGGER = 'ROSFinder_PATH_TRIGGER' -# -# def __init__(self, path_entry): -# self.logger = logging.getLogger(__name__) -# self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) -# if path_entry != self.PATH_TRIGGER: -# self.logger.debug('ROSImportFinder does not work for %s' % path_entry) -# raise ImportError() -# -# self.loaders = { -# '.srv': ROSLoader(genpy.generator.SrvGenerator(), 'srv'), -# '.msg': ROSLoader(genpy.generator.MsgGenerator(), 'msg') -# } -# -# def find_module(self, fullname, path=None): -# print('ROSImportFinder looking for "%s"' % fullname) -# return None - -else: - class ROSImportFinder(object): - """Find ROS message/service modules""" - def __init__(self, path_entry=None): - # First we need to skip all the cases that we are not concerned with - - # path_entry contains the path where the finder has been instantiated... - if not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists(os.path.join(path_entry, 'srv')): - raise ImportError # we raise if we cannot find msg or srv folder - - # Then we can do the initialisation - self.logger = logging.getLogger(__name__) - self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) - - self.path_entry = path_entry - - - # This is called for the first unknown (not loaded) name inside path - def find_module(self, name, path=None): - self.logger.debug('ROSImportFinder looking for "%s"' % name) - - # on 2.7 - path = path or self.path_entry - - # implementation inspired from pytest.rewrite - names = name.rsplit(".", 1) - lastname = names[-1] - - - if not os.path.exists(os.path.join(path, lastname)): - raise ImportError - elif os.path.isdir(os.path.join(path, lastname)): - - rosf = [f for f in os.listdir(os.path.join(path, lastname)) if os.path.splitext(f)[-1] in ['.msg', '.srv']] - - if rosf: - return ROSLoader( - msgsrv_files=rosf, - path_entry=os.path.join(path, lastname), - package=name, - outdir_pkg=path, # TODO: if we cannot write into source, use tempfile - #includepath=, - #ns_pkg=, - ) - - # # package case - # for root, dirs, files in os.walk(path, topdown=True): - # - # #rosmsg_generator.genmsgsrv_py(msgsrv_files=[f for f in files if ], package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) - # - # # generated_msg = genmsg_py(msg_files=[f for f in files if f.endswith('.msg')], - # # package=package, - # # outdir_pkg=outdir_pkg, - # # includepath=includepath, - # # initpy=True) # we always create an __init__.py when called from here. - # # generated_srv = gensrv_py(srv_files=[f for f in files if f.endswith('.srv')], - # # package=package, - # # outdir_pkg=outdir_pkg, - # # includepath=includepath, - # # initpy=True) # we always create an __init__.py when called from here. - # - # for name in dirs: - # print(os.path.join(root, name)) - # # rosmsg_generator.genmsgsrv_py(msgsrv_files=msgsrvfiles, package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) - - elif os.path.isfile(os.path.join(path, lastname)): - # module case - return ROSLoader( - msgsrv_files=os.path.join(path, lastname), - path_entry=path, - package=name, - outdir_pkg=path, # TODO: if we cannot write into source, use tempfile - # includepath=, - # ns_pkg=, - ) - - return None - - - - - - # def find_module(self, name, path=None): - # """ - # Return the loader for the specified module. - # """ - # # Ref : https://www.python.org/dev/peps/pep-0302/#specification-part-1-the-importer-protocol - # - # # - # loader = None - # - # # path = path or sys.path - # # for p in path: - # # for f in os.listdir(p): - # # filename, ext = os.path.splitext(f) - # # # our modules generated from messages are always a leaf in import tree so we only care about this case - # # if ext in self.loaders.keys() and filename == name.split('.')[-1]: - # # loader = self.loaders.get(ext) - # # break # we found it. break out. - # # - # # return loader - # - # # implementation inspired from pytest.rewrite - # self.logger.debug("find_module called for: %s" % name) - # names = name.rsplit(".", 1) - # lastname = names[-1] - # pth = None - # if path is not None: - # # Starting with Python 3.3, path is a _NamespacePath(), which - # # causes problems if not converted to list. - # path = list(path) - # if len(path) == 1: - # pth = path[0] - # - # - # if pth is None: - # - # - # - # - # - # try: - # fd, fn, desc = imp.find_module(lastname, path) - # except ImportError: - # return None - # if fd is not None: - # fd.close() - # - # - # - # - # tp = desc[2] - # if tp == imp.PY_COMPILED: - # if hasattr(imp, "source_from_cache"): - # try: - # fn = imp.source_from_cache(fn) - # except ValueError: - # # Python 3 doesn't like orphaned but still-importable - # # .pyc files. - # fn = fn[:-1] - # else: - # fn = fn[:-1] - # elif tp != imp.PY_SOURCE: - # # Don't know what this is. - # return None - # else: - # fn = os.path.join(pth, name.rpartition(".")[2] + ".py") - # - # fn_pypath = py.path.local(fn) - # if not self._should_rewrite(name, fn_pypath, state): - # return None - # - # self._rewritten_names.add(name) - # - # # The requested module looks like a test file, so rewrite it. This is - # # the most magical part of the process: load the source, rewrite the - # # asserts, and load the rewritten source. We also cache the rewritten - # # module code in a special pyc. We must be aware of the possibility of - # # concurrent pytest processes rewriting and loading pycs. To avoid - # # tricky race conditions, we maintain the following invariant: The - # # cached pyc is always a complete, valid pyc. Operations on it must be - # # atomic. POSIX's atomic rename comes in handy. - # write = not sys.dont_write_bytecode - # cache_dir = os.path.join(fn_pypath.dirname, "__pycache__") - # if write: - # try: - # os.mkdir(cache_dir) - # except OSError: - # e = sys.exc_info()[1].errno - # if e == errno.EEXIST: - # # Either the __pycache__ directory already exists (the - # # common case) or it's blocked by a non-dir node. In the - # # latter case, we'll ignore it in _write_pyc. - # pass - # elif e in [errno.ENOENT, errno.ENOTDIR]: - # # One of the path components was not a directory, likely - # # because we're in a zip file. - # write = False - # elif e in [errno.EACCES, errno.EROFS, errno.EPERM]: - # state.trace("read only directory: %r" % fn_pypath.dirname) - # write = False - # else: - # raise - # cache_name = fn_pypath.basename[:-3] + PYC_TAIL - # pyc = os.path.join(cache_dir, cache_name) - # # Notice that even if we're in a read-only directory, I'm going - # # to check for a cached pyc. This may not be optimal... - # co = _read_pyc(fn_pypath, pyc, state.trace) - # if co is None: - # state.trace("rewriting %r" % (fn,)) - # source_stat, co = _rewrite_test(self.config, fn_pypath) - # if co is None: - # # Probably a SyntaxError in the test. - # return None - # if write: - # _make_rewritten_pyc(state, source_stat, pyc, co) - # else: - # state.trace("found cached rewritten pyc for %r" % (fn,)) - # self.modules[name] = co, pyc - # return self - -#_ros_finder_instance_obsolete_python = ROSImportFinder - - -def activate(): - #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? - sys.path_hooks.append(ROSImportFinder) - - # else: # older (trusty) version - # sys.path_hooks.append(_ros_finder_instance_obsolete_python) - - for hook in sys.path_hooks: - print('Path hook: {}'.format(hook)) - - -def deactivate(): - #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? - sys.path_hooks.remove(ROSImportFinder) - # else: # older (trusty) version - # sys.path_hooks.remove(_ros_finder_instance_obsolete_python) - - -# TODO : a meta finder could find a full ROS distro... diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py new file mode 100644 index 0000000..b0f5ed9 --- /dev/null +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -0,0 +1,267 @@ +from __future__ import absolute_import, division, print_function + +import contextlib +import importlib +import site + +from pyros_msgs.importer import rosmsg_generator + +""" +A module to setup custom importer for .msg and .srv files +Upon import, it will first find the .msg file, then generate the python module for it, then load it. + +TODO... +""" + +# We need to be extra careful with python versions +# Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module + +# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path +# Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) +import os +import sys + +# This will take the ROS distro version if ros has been setup +import genpy.generator +import genpy.generate_initpy + +import logging + + +def _mk_init_name(fullname): + """Return the name of the __init__ module for a given package name.""" + if fullname.endswith('.__init__'): + return fullname + return fullname + '.__init__' + + +def _get_key_name(fullname, db): + """Look in an open shelf for fullname or fullname.__init__, return the name found.""" + if fullname in db: + return fullname + init_name = _mk_init_name(fullname) + if init_name in db: + return init_name + return None + + + + + + + +if sys.version_info >= (3, 4): + + import importlib.abc + import importlib.machinery + + + class ROSLoader(importlib.abc.SourceLoader): + + def __init__(self, path_entry, msgsrv_files, package, outdir_pkg, includepath=None, ns_pkg=None): + + self.logger = logging.getLogger(__name__) + self.path_entry = path_entry + + self.msgsrv_files = msgsrv_files + self.package = package + self.outdir_pkg = outdir_pkg + + # TODO : we need to determine that from the loader + self.includepath = includepath + self.ns_pkg = ns_pkg + + def _generate(self, fullname): + + gen = rosmsg_generator.genmsgsrv_py( + msgsrv_files=self.msgsrv_files, + package=self.package, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + ns_pkg=self.ns_pkg + ) + return gen + + def get_source(self, fullname): + self.logger.debug('loading source for "%s" from msg' % fullname) + try: + + self._generate() + + # with shelve_context(self.path_entry) as db: + # key_name = _get_key_name(fullname, db) + # if key_name: + # return db[key_name] + # raise ImportError('could not find source for %s' % fullname) + + pass + + + except Exception as e: + self.logger.debug('could not load source:', e) + raise ImportError(str(e)) + + # defining this to benefit from backward compat import mechanism in python 3.X + def get_filename(self, name): + os.sep.join(name.split(".")) + '.' + self.ext + + # def _get_filename(self, fullname): + # # Make up a fake filename that starts with the path entry + # # so pkgutil.get_data() works correctly. + # return os.path.join(self.path_entry, fullname) + # + # # defining this to benefit from backward compat import mechanism in python 3.X + # def is_package(self, name): + # names = name.split(".") + # parent_idx = len(names) -1 + # # trying to find a parent already loaded + # while 0<= parent_idx < len(names) : + # if names[parent_idx] in sys.modules: # we found a parent, we can get his path and go back + # pass + # else: # parent not found, need to check its parent + # parent_idx-=1 + # + # + # else: # No loaded parent found, lets attempt to import it directly (searching in all sys.paths) + # + # pass + # + # + # return None # TODO : implement check + # + # def load_module(self, fullname): + # + # if fullname in sys.modules: + # self.logger.debug('reusing existing module from previous import of "%s"' % fullname) + # mod = sys.modules[fullname] + # else: + # self.logger.debug('creating a new module object for "%s"' % fullname) + # mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + # + # # Set a few properties required by PEP 302 + # mod.__file__ = self._get_filename(fullname) + # mod.__name__ = fullname + # mod.__path__ = self.path_entry + # mod.__loader__ = self + # mod.__package__ = '.'.join(fullname.split('.')[:-1]) + # + # if self.is_package(fullname): + # self.logger.debug('adding path for package') + # # Set __path__ for packages + # # so we can find the sub-modules. + # mod.__path__ = [self.path_entry] + # else: + # self.logger.debug('imported as regular module') + # + # source = self.get_source(fullname) + # + # self.logger.debug('execing source...') + # exec(source, mod.__dict__) + # self.logger.debug('done') + # return mod + +else: + + class ROSLoader(object): + + def __init__(self, path_entry, msgsrv_files, package, outdir_pkg, includepath=None, ns_pkg=None): + + self.logger = logging.getLogger(__name__) + self.path_entry = path_entry + + self.msgsrv_files = msgsrv_files + self.package = package + self.outdir_pkg = outdir_pkg + + # TODO : we need to determine that from the loader + self.includepath = includepath + self.ns_pkg = ns_pkg + + def _generate(self, fullname): + + gen = rosmsg_generator.genmsgsrv_py( + msgsrv_files=self.msgsrv_files, + package=self.package, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + ns_pkg=self.ns_pkg + ) + return gen + + def get_source(self, fullname): + self.logger.debug('loading source for "%s" from msg' % fullname) + try: + + self._generate() + + # with shelve_context(self.path_entry) as db: + # key_name = _get_key_name(fullname, db) + # if key_name: + # return db[key_name] + # raise ImportError('could not find source for %s' % fullname) + + pass + + + except Exception as e: + self.logger.debug('could not load source:', e) + raise ImportError(str(e)) + + # defining this to benefit from backward compat import mechanism in python 3.X + def get_filename(self, name): + os.sep.join(name.split(".")) + '.' + self.ext + + # def _get_filename(self, fullname): + # # Make up a fake filename that starts with the path entry + # # so pkgutil.get_data() works correctly. + # return os.path.join(self.path_entry, fullname) + # + # # defining this to benefit from backward compat import mechanism in python 3.X + # def is_package(self, name): + # names = name.split(".") + # parent_idx = len(names) -1 + # # trying to find a parent already loaded + # while 0<= parent_idx < len(names) : + # if names[parent_idx] in sys.modules: # we found a parent, we can get his path and go back + # pass + # else: # parent not found, need to check its parent + # parent_idx-=1 + # + # + # else: # No loaded parent found, lets attempt to import it directly (searching in all sys.paths) + # + # pass + # + # + # return None # TODO : implement check + # + # def load_module(self, fullname): + # + # if fullname in sys.modules: + # self.logger.debug('reusing existing module from previous import of "%s"' % fullname) + # mod = sys.modules[fullname] + # else: + # self.logger.debug('creating a new module object for "%s"' % fullname) + # mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + # + # # Set a few properties required by PEP 302 + # mod.__file__ = self._get_filename(fullname) + # mod.__name__ = fullname + # mod.__path__ = self.path_entry + # mod.__loader__ = self + # mod.__package__ = '.'.join(fullname.split('.')[:-1]) + # + # if self.is_package(fullname): + # self.logger.debug('adding path for package') + # # Set __path__ for packages + # # so we can find the sub-modules. + # mod.__path__ = [self.path_entry] + # else: + # self.logger.debug('imported as regular module') + # + # source = self.get_source(fullname) + # + # self.logger.debug('execing source...') + # exec(source, mod.__dict__) + # self.logger.debug('done') + # return mod diff --git a/pyros_msgs/importer/rosmsg_metafinder.py b/pyros_msgs/importer/rosmsg_metafinder.py new file mode 100644 index 0000000..3f7d328 --- /dev/null +++ b/pyros_msgs/importer/rosmsg_metafinder.py @@ -0,0 +1,192 @@ +from __future__ import absolute_import, division, print_function + +import contextlib +import importlib +import site + +from pyros_msgs.importer import rosmsg_generator + +""" +A module to setup custom importer for .msg and .srv files +Upon import, it will first find the .msg file, then generate the python module for it, then load it. + +TODO... +""" + +# We need to be extra careful with python versions +# Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module + +# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path +# Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) +import os +import sys + +# This will take the ROS distro version if ros has been setup +import genpy.generator +import genpy.generate_initpy + +import logging + + + + + +if sys.version_info >= (3, 4): + + import importlib.abc + import importlib.machinery + + class ROSImportMetaFinder(importlib.abc.MetaPathFinder): + def __init__(self, *workspaces): + """ + :param workspaces: can be a devel or install workspace (including a distro install directory), but also a directory containing packages (like a source workspace) + These should all work, without catkin build necessary. + """ + super(ROSImportMetaFinder, self).__init__() + # TODO : prevent that when we are in virtualenv and we have not allowed system site packages + + self.src_rospkgs = [] + + broken_workspaces = [] + for w in workspaces: + # Adding the share path (where we can find all packages and their messages) + share_path = os.path.join(w, 'share') + if os.path.exists(share_path): + self.share_path = share_path + else: # this is not a workspace, maybe we are expected to get the package directly from source ? + found_one = False + for root, dirs, files in os.walk(w, topdown=False): + if 'package.xml' in files: # we have found a ros package + self.src_rospkgs.append(root) + found_one = True + if not found_one: + broken_workspaces.append(w) + + raise ImportError + + python_libpath = os.path.join(w, 'lib', 'python' + sys.version_info.major + '.' + sys.version_info.minor) + if os.path.exists(python_libpath): + # adding python site directories for the ROS distro (to find python modules as usual with ROS) + if os.path.exists(os.path.join(python_libpath, 'dist-packages')): + site.addsitedir(os.path.join(python_libpath, 'dist-packages')) + if os.path.exists(os.path.join(python_libpath, 'site-packages')): + site.addsitedir(os.path.join(python_libpath, 'site-packages')) + else: # this is not a workspace, maybe we are expected to get the package directly from source ? + + for root, dirs, files in os.walk(w, topdown=False): + if 'package.xml' in files: # we have found a ros package + self.src_rospkgs.append(root) + + raise ImportError + + + + + def find_spec(self, fullname, path, target=None): + """ + :param fullname: the name of the package we are trying to import + :param path: path we we expect to find it (can be None) + :param target: what we plan to do with it + :return: + """ + # TODO: read PEP 420 :) + last_mile = fullname.split('.')[-1] + + path = path or sys.path + + # TODO: we should handle different structure of ROS packages (installed or not) + #if (os.path.exists) + + # TODO : similar to pyros-setup ? + # + setup rosmsg importers ? + if (not os.path.exists(os.path.join(last_mile, 'package.xml')) or + False): + raise ImportError + + for p in path: + fullpath = os.path.join(p, last_mile) + init_fullpath = os.path.join(fullpath, '__init__.ay') + module_fullpath = fullpath + '.ay' + + if os.path.isdir(fullpath) and os.path.exists(init_fullpath): + return importlib.machinery.ModuleSpec( + fullname, + loader, + origin=init_fullpath + ) + + else: + if os.path.exists(module_fullpath): + return importlib.machinery.ModuleSpec( + fullname, + loader, + origin=module_fullpath + ) + + return None + +else: + passb # TODO + + +# Useless ? +#_ros_finder_instance_obsolete_python = ROSImportFinder + +ros_distro_finder = None + + +def activate(rosdistro_path=None, *workspaces): + global ros_distro_finder + if rosdistro_path is None: # autodetect most recent installed distro + if os.path.exists('/opt/ros/lunar'): + rosdistro_path = '/opt/ros/lunar' + elif os.path.exists('/opt/ros/kinetic'): + rosdistro_path = '/opt/ros/kinetic' + elif os.path.exists('/opt/ros/jade'): + rosdistro_path = '/opt/ros/jade' + elif os.path.exists('/opt/ros/indigo'): + rosdistro_path = '/opt/ros/indigo' + else: + raise ImportError( + "No ROS distro detected on this system. Please specify the path when calling ROSImportMetaFinder()") + + ros_distro_finder = ROSImportMetaFinder(rosdistro_path, *workspaces) + sys.meta_path.append(ros_distro_finder) + + #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? + + # We need to be before FileFinder to be able to find our (non .py[c]) files + # inside, maybe already imported, python packages... + sys.path_hooks.insert(1, ROSImportFinder) + + # else: # older (trusty) version + # sys.path_hooks.append(_ros_finder_instance_obsolete_python) + + for hook in sys.path_hooks: + print('Path hook: {}'.format(hook)) + + # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths + if paths: + sys.path.append(*paths) + + +def deactivate(*paths): + """ CAREFUL : even if we remove our path_hooks, the created finder are still cached in sys.path_importer_cache.""" + #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? + sys.path_hooks.remove(ROSImportFinder) + # else: # older (trusty) version + # sys.path_hooks.remove(_ros_finder_instance_obsolete_python) + if paths: + sys.path.remove(*paths) + + sys.meta_path.remove(ros_distro_finder) + + +@contextlib.contextmanager +def ROSImportContext(*paths): + activate(*paths) + yield + deactivate(*paths) + + +# TODO : a meta finder could find a full ROS distro... diff --git a/pyros_msgs/importer/tests/__init__.py b/pyros_msgs/importer/tests/__init__.py index 71eff8b..8b8e09e 100644 --- a/pyros_msgs/importer/tests/__init__.py +++ b/pyros_msgs/importer/tests/__init__.py @@ -1 +1,23 @@ -# TODO ! +from __future__ import absolute_import, division, print_function +# +# import os +# try: +# # Using std_msgs directly if ROS has been setup (while using from ROS pkg) +# import std_msgs +# +# except ImportError: +# +# # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) +# +# import site +# site.addsitedir(os.path.join( +# os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), +# 'ros-site' +# )) +# +# import std_msgs +# +# # Note we do not want to use pyros_setup here. +# # We do not want to do a full ROS setup, only import specific packages. +# # If needed it should have been done before (loading a parent package). +# # this handle the case where we want to be independent of any underlying ROS system. diff --git a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py b/pyros_msgs/importer/tests/test_rosmsg_import_pkg_absolute.py similarity index 74% rename from pyros_msgs/importer/tests/test_rosmsg_generator_import.py rename to pyros_msgs/importer/tests/test_rosmsg_import_pkg_absolute.py index 421e594..bac8b05 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_generator_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_import_pkg_absolute.py @@ -35,7 +35,7 @@ import importlib # Importing importer module -from pyros_msgs.importer import rosmsg_importer +from pyros_msgs.importer import rosmsg_finder # importlib # https://pymotw.com/3/importlib/index.html @@ -151,40 +151,66 @@ def print_importers(): class TestImportAnotherMsg(unittest.TestCase): + + rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') + @classmethod def setUpClass(cls): - # trying to avoid cache pollution of consecutive tests.. - if hasattr(importlib, 'invalidate_caches'): - importlib.invalidate_caches() - rosmsg_importer.activate() + # We need to be before FileFinder to be able to find our (non .py[c]) files + # inside, maybe already imported, python packages... + sys.path_hooks.insert(1, rosmsg_finder.ROSImportFinder) + sys.path.append(cls.rosdeps_path) @classmethod def tearDownClass(cls): - rosmsg_importer.deactivate() + # CAREFUL : Even though we remove the path from sys.path, + # initialized finders will remain in sys.path_importer_cache + sys.path.remove(cls.rosdeps_path) def test_import_absolute(self): print_importers() # Verify that files exists and are importable import std_msgs.msg.Bool as msg_bool - assert msg_bool is not None + self.assertTrue(msg_bool is not None) def test_import_from(self): + print_importers() # Verify that files exists and are importable - from std_msgs.msg import Bool as msg_bool - - assert msg_bool is not None + try: + # Using std_msgs directly if ROS has been setup (while using from ROS pkg) + from std_msgs.msg import Bool as msg_bool + except ImportError: + # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) + import site + site.addsitedir(os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), + 'ros-site' + )) + from std_msgs.msg import Bool as msg_bool + + self.assertTrue(msg_bool is not None) @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") def test_importlib_import_absolute(self): # Verify that files exists and are importable - msg_bool = importlib.__import__('std_msgs.msg.Bool',) + try: + # Using std_msgs directly if ROS has been setup (while using from ROS pkg) + msg_bool = importlib.__import__('std_msgs.msg.Bool', ) + except ImportError: + # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) + import site + site.addsitedir(os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), + 'ros-site' + )) + msg_bool = importlib.__import__('std_msgs.msg.Bool',) assert msg_bool is not None @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), - reason="importlib does not have attribute import_module") + reason="importlib does not have attribute find_loader or load_module") def test_importlib_loadmodule_absolute(self): # Verify that files exists and are dynamically importable pkg_list = 'std_msgs.msg.Bool'.split('.')[:-1] @@ -218,71 +244,6 @@ def test_importlib_importmodule_absolute(self): assert msg_bool is not None -class TestImportSameMsg(unittest.TestCase): - @classmethod - def setUpClass(cls): - # trying to avoid cache pollution of consecutive tests.. - if hasattr(importlib, 'invalidate_caches'): - importlib.invalidate_caches() - rosmsg_importer.activate() - - @classmethod - def tearDownClass(cls): - rosmsg_importer.deactivate() - - def test_import_absolute(self): - print_importers() - # Verify that files exists and are importable - import pyros_msgs.importer.tests.msg.TestMsg as msg_mod - - assert msg_mod is not None - - def test_import_relative(self): - print_importers() - # Verify that files exists and are importable - from .msg import TestMsg as msg_mod - - assert msg_mod is not None - - @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_absolute(self): - # Verify that files exists and are importable - msg_mod = importlib.__import__('pyros_msgs.importer.tests.msg.TestMsg',) - - assert msg_mod is not None - - @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute import_module") - def test_importlib_loadmodule_absolute(self): - # Verify that files exists and are dynamically importable - pkg_list = 'pyros_msgs.importer.tests.msg.TestMsg'.split('.')[:-1] - mod_list = 'pyros_msgs.importer.tests.msg.TestMsg'.split('.')[1:] - pkg = None - for pkg_name, mod_name in zip(pkg_list, mod_list): - pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) - pkg = pkg_loader.load_module(mod_name) - - msg_mod = pkg - - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(msg_mod) - else: - pass - - # TODO : dynamic using specs (python 3.5) - - @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") - def test_importlib_importmodule_absolute(self): - # Verify that files exists and are dynamically importable - msg_mod = importlib.import_module('pyros_msgs.importer.tests.msg.TestMsg') - - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(msg_mod) - else: - pass - - # def test_importlib_msg_module_relative(): # # Verify that files exists and are importable diff --git a/tests/importer/test_rosmsg_import_msgmod_relative.py b/tests/importer/test_rosmsg_import_msgmod_relative.py new file mode 100644 index 0000000..a9569be --- /dev/null +++ b/tests/importer/test_rosmsg_import_msgmod_relative.py @@ -0,0 +1,272 @@ +from __future__ import absolute_import, division, print_function +""" +Testing executing rosmsg_import outside of the package, otherwise the package is already imported. +""" + +import os +import sys +import runpy +import logging.config + +logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'default': { + 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + } + }, + 'root': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, +}) + +# Since test frameworks (like pytest) play with the import machinery, we cannot use it here... +import unittest + +# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path + +import importlib + +# Importing importer module +from pyros_msgs.importer import rosmsg_finder + +# importlib +# https://pymotw.com/3/importlib/index.html +# https://pymotw.com/2/importlib/index.html + +# if sys.version_info > (3, 4): +# import importlib.util +# # REF : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module +# # Useful to display details when debugging... +# def import_module(name, package=None): +# """An approximate implementation of import.""" +# absolute_name = importlib.util.resolve_name(name, package) +# try: +# return sys.modules[absolute_name] +# except KeyError: +# pass +# +# path = None +# if '.' in absolute_name: +# parent_name, _, child_name = absolute_name.rpartition('.') +# parent_module = import_module(parent_name) +# path = parent_module.spec.submodule_search_locations # this currently breaks (probably because of pytest custom importer ? ) +# for finder in sys.meta_path: +# spec = finder.find_spec(absolute_name, path) +# if spec is not None: +# break +# else: +# raise ImportError('No module named {absolute_name!r}'.format(**locals())) +# module = importlib.util.module_from_spec(spec) +# spec.loader.exec_module(module) +# sys.modules[absolute_name] = module +# if path is not None: +# setattr(parent_module, child_name, module) +# return module + + +# if sys.version_info > (3, 4): +# import importlib.util +# +# # Adapted from : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module +# # Useful to debug details... +# def import_module(name, package=None): +# # using find_spec to use our finder +# spec = importlib.util.find_spec(name, package) +# +# # path = None +# # if '.' in absolute_name: +# # parent_name, _, child_name = absolute_name.rpartition('.') +# # # recursive import call +# # parent_module = import_module(parent_name) +# # # getting the path instead of relying on spec (not managed by pytest it seems...) +# # path = [os.path.join(p, child_name) for p in parent_module.__path__ if os.path.exists(os.path.join(p, child_name))] +# +# # getting spec with importlib.util (instead of meta path finder to avoid uncompatible pytest finder) +# #spec = importlib.util.spec_from_file_location(absolute_name, path[0]) +# parent_module = None +# child_name = None +# if spec is None: +# # We didnt find anything, but this is expected on ros packages that haven't been generated yet +# if '.' in name: +# parent_name, _, child_name = name.rpartition('.') +# # recursive import call +# parent_module = import_module(parent_name) +# # we can check if there is a module in there.. +# path = [os.path.join(p, child_name) +# for p in parent_module.__path__._path +# if os.path.exists(os.path.join(p, child_name))] +# # we attempt to get the spec from the first found location +# while path and spec is None: +# spec = importlib.util.spec_from_file_location(name, path[0]) +# path[:] = path[1:] +# else: +# raise ImportError +# +# # checking again in case spec has been modified +# if spec is not None: +# if spec.name not in sys.modules: +# module = importlib.util.module_from_spec(spec) +# spec.loader.exec_module(module) +# sys.modules[name] = module +# if parent_module is not None and child_name is not None: +# setattr(parent_module, child_name, sys.modules[name]) +# return sys.modules[name] +# else: +# raise ImportError +# else: +# def import_module(name, package=None): +# return importlib.import_module(name=name, package=package) + + +# +# Note : we cannot assume anything about import implementation (different python version, different version of pytest) +# => we need to test them all... +# + +# HACK to fix spec from pytest +# @property +# def spec(): +# importlib.util.spec_from_file_location(__file__) + + +def print_importers(): + import sys + import pprint + + print('PATH:'), + pprint.pprint(sys.path) + print() + print('IMPORTERS:') + for name, cache_value in sys.path_importer_cache.items(): + name = name.replace(sys.prefix, '...') + print('%s: %r' % (name, cache_value)) + + +class TestImportRelativeMsgModule(unittest.TestCase): + """Testing custom ROS importer with path already existing and into sys.path""" + + @classmethod + def setUpClass(cls): + # We need to be before FileFinder to be able to find our (non .py[c]) files + # inside, maybe already imported, python packages... + sys.path_hooks.insert(1, rosmsg_finder.ROSFileFinder) + # we do not need to append any path (since we rely on paths already present, and package already found) + + @classmethod + def tearDownClass(cls): + # CAREFUL : Even though we remove the path from sys.path, + # initialized finders will remain in sys.path_importer_cache + sys.path_hooks.remove(rosmsg_finder.ROSFileFinder) + + def test_import_absolute_module(self): + print_importers() + # Verify that files exists and are importable + import pyros_msgs.importer.tests.msg.TestMsg as msg_mod + + self.assertTrue(msg_mod is not None) + + @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + def test_importlib_import_absolute(self): + # Verify that files exists and are importable + msg_mod = importlib.__import__('pyros_msgs.importer.tests.msg.TestMsg',) + + self.assertTrue(msg_mod is not None) + + @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") + def test_importlib_loadmodule_absolute(self): + # Verify that files exists and are dynamically importable + pkg_list = 'pyros_msgs.importer.tests.msg.TestMsg'.split('.')[:-1] + mod_list = 'pyros_msgs.importer.tests.msg.TestMsg'.split('.')[1:] + pkg = None + for pkg_name, mod_name in zip(pkg_list, mod_list): + pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) + pkg = pkg_loader.load_module(mod_name) + + msg_mod = pkg + + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_mod) + else: + pass + + # TODO : dynamic using specs (python 3.5) + + @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") + def test_importlib_importmodule_absolute(self): + # Verify that files exists and are dynamically importable + msg_mod = importlib.import_module('pyros_msgs.importer.tests.msg.TestMsg') + + self.assertTrue(msg_mod is not None) + + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_mod) + else: + pass + + self.assertTrue(msg_mod is not None) + + +# def test_importlib_msg_module_relative(): +# # Verify that files exists and are importable +# # TODO +# # if hasattr(importlib, 'find_loader'): # recent version of import lib +# # +# # pkg_loader = importlib.find_loader('test_gen_msgs') +# # pkg = pkg_loader.load_module() +# # +# # subpkg_loader = importlib.find_loader('msg', pkg.__path__) +# # subpkg = subpkg_loader.load_module() +# # +# # loader = importlib.find_loader('TestMsg', subpkg.__path__) +# # m = subpkg_loader.load_module() +# # +# # else: # old version +# msg_mod = importlib.import_module('.msg.TestMsg', package=__package__) + + + + +def test_importlib_srv_module(): + pass + # TODO + # # Verify that files exists and are importable + # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') + + +# imp https://pymotw.com/2/imp/index.html +# TODO + +# def test_imp_msg_module(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.msg.TestMsg') +# +# +# def test_imp_msg_pkg(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.msg') +# +# +# def test_imp_srv_module(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.srv.TestSrv') +# +# +# def test_imp_srv_pkg(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.srv') + + +if __name__ == '__main__': + unittest.main() + #import nose + #nose.main() diff --git a/tests/importer/test_rosmsg_import_msgpkg_relative.py b/tests/importer/test_rosmsg_import_msgpkg_relative.py new file mode 100644 index 0000000..03b0a14 --- /dev/null +++ b/tests/importer/test_rosmsg_import_msgpkg_relative.py @@ -0,0 +1,275 @@ +from __future__ import absolute_import, division, print_function +""" +Testing executing rosmsg_import outside of the package, otherwise the package is already imported. +""" + +import os +import sys +import runpy +import logging.config + +logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'default': { + 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + } + }, + 'root': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, +}) + +# Since test frameworks (like pytest) play with the import machinery, we cannot use it here... +import unittest + +# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path + +import importlib + +# Importing importer module +from pyros_msgs.importer import rosmsg_finder + +# importlib +# https://pymotw.com/3/importlib/index.html +# https://pymotw.com/2/importlib/index.html + +# if sys.version_info > (3, 4): +# import importlib.util +# # REF : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module +# # Useful to display details when debugging... +# def import_module(name, package=None): +# """An approximate implementation of import.""" +# absolute_name = importlib.util.resolve_name(name, package) +# try: +# return sys.modules[absolute_name] +# except KeyError: +# pass +# +# path = None +# if '.' in absolute_name: +# parent_name, _, child_name = absolute_name.rpartition('.') +# parent_module = import_module(parent_name) +# path = parent_module.spec.submodule_search_locations # this currently breaks (probably because of pytest custom importer ? ) +# for finder in sys.meta_path: +# spec = finder.find_spec(absolute_name, path) +# if spec is not None: +# break +# else: +# raise ImportError('No module named {absolute_name!r}'.format(**locals())) +# module = importlib.util.module_from_spec(spec) +# spec.loader.exec_module(module) +# sys.modules[absolute_name] = module +# if path is not None: +# setattr(parent_module, child_name, module) +# return module + + +# if sys.version_info > (3, 4): +# import importlib.util +# +# # Adapted from : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module +# # Useful to debug details... +# def import_module(name, package=None): +# # using find_spec to use our finder +# spec = importlib.util.find_spec(name, package) +# +# # path = None +# # if '.' in absolute_name: +# # parent_name, _, child_name = absolute_name.rpartition('.') +# # # recursive import call +# # parent_module = import_module(parent_name) +# # # getting the path instead of relying on spec (not managed by pytest it seems...) +# # path = [os.path.join(p, child_name) for p in parent_module.__path__ if os.path.exists(os.path.join(p, child_name))] +# +# # getting spec with importlib.util (instead of meta path finder to avoid uncompatible pytest finder) +# #spec = importlib.util.spec_from_file_location(absolute_name, path[0]) +# parent_module = None +# child_name = None +# if spec is None: +# # We didnt find anything, but this is expected on ros packages that haven't been generated yet +# if '.' in name: +# parent_name, _, child_name = name.rpartition('.') +# # recursive import call +# parent_module = import_module(parent_name) +# # we can check if there is a module in there.. +# path = [os.path.join(p, child_name) +# for p in parent_module.__path__._path +# if os.path.exists(os.path.join(p, child_name))] +# # we attempt to get the spec from the first found location +# while path and spec is None: +# spec = importlib.util.spec_from_file_location(name, path[0]) +# path[:] = path[1:] +# else: +# raise ImportError +# +# # checking again in case spec has been modified +# if spec is not None: +# if spec.name not in sys.modules: +# module = importlib.util.module_from_spec(spec) +# spec.loader.exec_module(module) +# sys.modules[name] = module +# if parent_module is not None and child_name is not None: +# setattr(parent_module, child_name, sys.modules[name]) +# return sys.modules[name] +# else: +# raise ImportError +# else: +# def import_module(name, package=None): +# return importlib.import_module(name=name, package=package) + + +# +# Note : we cannot assume anything about import implementation (different python version, different version of pytest) +# => we need to test them all... +# + +# HACK to fix spec from pytest +# @property +# def spec(): +# importlib.util.spec_from_file_location(__file__) + + +def print_importers(): + import sys + import pprint + + print('PATH:'), + pprint.pprint(sys.path) + print() + print('IMPORTERS:') + for name, cache_value in sys.path_importer_cache.items(): + name = name.replace(sys.prefix, '...') + print('%s: %r' % (name, cache_value)) + + +class TestImportRelativeMsgPkg(unittest.TestCase): + """Testing custom ROS importer with path already existing and into sys.path""" + + @classmethod + def setUpClass(cls): + # We need to be before FileFinder to be able to find our (non .py[c]) files + # inside, maybe already imported, python packages... + sys.path_hooks.insert(1, rosmsg_finder.ROSFileFinder) + # we do not need to append any path (since we rely on paths already present, and package already found) + + @classmethod + def tearDownClass(cls): + # CAREFUL : Even though we remove the path from sys.path, + # initialized finders will remain in sys.path_importer_cache + sys.path_hooks.remove(rosmsg_finder.ROSFileFinder) + + def test_import_absolute_pkg(self): + print_importers() + # Verify that files exists and are importable + # Note : this requires a metapath finder + import pyros_msgs.importer.tests.msg as msg_pkg + self.assertTrue(msg_pkg is not None) + + msg_mod = msg_pkg.TestMsg + self.assertTrue(msg_mod is not None) + + @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + def test_importlib_import_absolute(self): + # Verify that files exists and are importable + msg_pkg = importlib.__import__('pyros_msgs.importer.tests.msg',) + + self.assertTrue(msg_pkg is not None) + + @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") + def test_importlib_loadmodule_absolute(self): + # Verify that files exists and are dynamically importable + pkg_list = 'pyros_msgs.importer.tests.msg'.split('.')[:-1] + mod_list = 'pyros_msgs.importer.tests.msg'.split('.')[1:] + pkg = None + for pkg_name, mod_name in zip(pkg_list, mod_list): + pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) + pkg = pkg_loader.load_module(mod_name) + + msg_mod = pkg + + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_mod) + else: + pass + + # TODO : dynamic using specs (python 3.5) + + @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") + def test_importlib_importmodule_absolute(self): + # Verify that files exists and are dynamically importable + msg_pkg = importlib.import_module('pyros_msgs.importer.tests.msg') + + self.assertTrue(msg_pkg is not None) + + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(msg_pkg) + else: + pass + + self.assertTrue(msg_pkg is not None) + + +# def test_importlib_msg_module_relative(): +# # Verify that files exists and are importable +# # TODO +# # if hasattr(importlib, 'find_loader'): # recent version of import lib +# # +# # pkg_loader = importlib.find_loader('test_gen_msgs') +# # pkg = pkg_loader.load_module() +# # +# # subpkg_loader = importlib.find_loader('msg', pkg.__path__) +# # subpkg = subpkg_loader.load_module() +# # +# # loader = importlib.find_loader('TestMsg', subpkg.__path__) +# # m = subpkg_loader.load_module() +# # +# # else: # old version +# msg_mod = importlib.import_module('.msg.TestMsg', package=__package__) + + + + +def test_importlib_srv_module(): + pass + # TODO + # # Verify that files exists and are importable + # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') + + +# imp https://pymotw.com/2/imp/index.html +# TODO + +# def test_imp_msg_module(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.msg.TestMsg') +# +# +# def test_imp_msg_pkg(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.msg') +# +# +# def test_imp_srv_module(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.srv.TestSrv') +# +# +# def test_imp_srv_pkg(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.srv') + + +if __name__ == '__main__': + unittest.main() + #import nose + #nose.main() From 52b108fb7f18a6319c6ae8925255cb9b7b67ee3f Mon Sep 17 00:00:00 2001 From: AlexV Date: Tue, 20 Jun 2017 20:16:34 +0900 Subject: [PATCH 07/28] WIP python code generation starting to work... --- pyros_msgs/importer/rosmsg_finder.py | 41 ++++++++++-- pyros_msgs/importer/rosmsg_loader.py | 95 +++++++++++++++++++++++----- 2 files changed, 114 insertions(+), 22 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py index 43054ed..98bff7e 100644 --- a/pyros_msgs/importer/rosmsg_finder.py +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -21,13 +21,9 @@ import os import sys -# This will take the ROS distro version if ros has been setup -import genpy.generator -import genpy.generate_initpy - import logging - +from .rosmsg_loader import ROSMsgLoader, ROSSrvLoader if sys.version_info >= (3, 4): @@ -38,13 +34,20 @@ class ROSFileFinder(importlib.machinery.FileFinder): - def __init__(self, path, *loader_details): + def __init__(self, path): """ Finder to get ROS specific files and directories (message and services files). It need to be inserted in sys.path_hooks before FileFinder, since these are Files but not python ones. :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) """ + + # Declaring our loaders and the different extensions + loader_details = [ + (ROSMsgLoader, ['.msg']), + (ROSSrvLoader, ['.srv']), + ] + super(ROSFileFinder, self).__init__(path, *loader_details) # First we need to skip all the cases that we are not concerned with @@ -100,13 +103,37 @@ def __init__(self, path, *loader_details): self.logger.debug('Checking ROSFileFinder support for %s' % path) + def _get_spec(self, loader_class, fullname, path, smsl, target): + loader = loader_class(fullname, path) + return importlib.util.spec_from_file_location(fullname, path, loader=loader, + submodule_search_locations=smsl) + def find_spec(self, fullname, target=None): - print('ROSImportFinder looking for "%s"' % fullname) """ + Try to find a spec for the specified module. + Returns the matching spec, or None if not found. :param fullname: the name of the package we are trying to import :param target: what we plan to do with it :return: """ + + print('ROSFileFinder looking for "%s"' % fullname) + tail_module = fullname.rpartition('.')[2] + + # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. + base_path = os.path.join(self.path, tail_module) + if os.path.isdir(base_path): + found_one = False + loader = None + for suffix, loader_class in self._loaders: + loader = loader_class if [f for f in os.listdir(base_path) if f.endswith(suffix)] else loader + # DO we need one or two loaders ? (package logic is same, but msg or srv differs after...) + spec = self._get_spec(loader, fullname, base_path, [base_path], target) + else: + # we use default behavior + spec = super(ROSFileFinder, self).find_spec(fullname=fullname, target=target) + return spec + # TODO: read PEP 420 :) last_mile = fullname.split('.')[-1] diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index b0f5ed9..1cd7918 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -3,6 +3,7 @@ import contextlib import importlib import site +import tempfile from pyros_msgs.importer import rosmsg_generator @@ -47,40 +48,48 @@ def _get_key_name(fullname, db): - - - if sys.version_info >= (3, 4): import importlib.abc import importlib.machinery + # To do only once when starting up + rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport') + if os.path.exists(rosimport_path): + os.removedirs(rosimport_path) + os.makedirs(rosimport_path) - class ROSLoader(importlib.abc.SourceLoader): + class ROSMsgLoader(importlib.abc.SourceLoader): - def __init__(self, path_entry, msgsrv_files, package, outdir_pkg, includepath=None, ns_pkg=None): + + def __init__(self, fullname, path): self.logger = logging.getLogger(__name__) - self.path_entry = path_entry + self.path = path - self.msgsrv_files = msgsrv_files - self.package = package - self.outdir_pkg = outdir_pkg + self.msgsrv_files = [f for f in os.listdir(self.path) if f.endswith('.msg')] - # TODO : we need to determine that from the loader - self.includepath = includepath - self.ns_pkg = ns_pkg + self.rospackage = fullname.partition('.')[0] - def _generate(self, fullname): + self.outdir_pkg = tempfile.mkdtemp(prefix=self.rospackage, dir=rosimport_path) + + # TODO : we need to determine that from the loader + self.includepath = [] + self.ns_pkg = False # doesnt seem needed yet and is overall safer... gen = rosmsg_generator.genmsgsrv_py( msgsrv_files=self.msgsrv_files, - package=self.package, + package=self.rospackage, outdir_pkg=self.outdir_pkg, includepath=self.includepath, ns_pkg=self.ns_pkg ) - return gen + + def get_data(self, path): + """Returns the bytes from the source code (will be used to generate bytecode)""" + self.get_filename() + return None + def get_source(self, fullname): self.logger.debug('loading source for "%s" from msg' % fullname) @@ -103,6 +112,11 @@ def get_source(self, fullname): # defining this to benefit from backward compat import mechanism in python 3.X def get_filename(self, name): + """ + Deterministic temporary filename. + :param name: + :return: + """ os.sep.join(name.split(".")) + '.' + self.ext # def _get_filename(self, fullname): @@ -160,6 +174,57 @@ def get_filename(self, name): # self.logger.debug('done') # return mod + class ROSSrvLoader(importlib.abc.SourceLoader): + + def __init__(self, path_entry, msgsrv_files, package, outdir_pkg, includepath=None, ns_pkg=None): + + self.logger = logging.getLogger(__name__) + self.path_entry = path_entry + + self.msgsrv_files = msgsrv_files + self.package = package + self.outdir_pkg = outdir_pkg + + # TODO : we need to determine that from the loader + self.includepath = includepath + self.ns_pkg = ns_pkg + + def _generate(self, fullname): + + gen = rosmsg_generator.genmsgsrv_py( + msgsrv_files=self.msgsrv_files, + package=self.package, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + ns_pkg=self.ns_pkg + ) + return gen + + def get_source(self, fullname): + self.logger.debug('loading source for "%s" from msg' % fullname) + try: + + self._generate() + + # with shelve_context(self.path_entry) as db: + # key_name = _get_key_name(fullname, db) + # if key_name: + # return db[key_name] + # raise ImportError('could not find source for %s' % fullname) + + pass + + + except Exception as e: + self.logger.debug('could not load source:', e) + raise ImportError(str(e)) + + # defining this to benefit from backward compat import mechanism in python 3.X + def get_filename(self, name): + os.sep.join(name.split(".")) + '.' + self.ext + + + else: class ROSLoader(object): From 6c21efaef8ab14527af02b0a573e75f278c3e989 Mon Sep 17 00:00:00 2001 From: AlexV Date: Wed, 21 Jun 2017 19:35:29 +0900 Subject: [PATCH 08/28] WIP import tests starting to pass ! --- pyros_msgs/importer/rosmsg_finder.py | 132 +++----------- pyros_msgs/importer/rosmsg_loader.py | 170 +++++++----------- .../test_rosmsg_import_msgmod_relative.py | 5 +- .../test_rosmsg_import_msgpkg_relative.py | 9 +- 4 files changed, 99 insertions(+), 217 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py index 98bff7e..26a1948 100644 --- a/pyros_msgs/importer/rosmsg_finder.py +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -43,70 +43,16 @@ def __init__(self, path): """ # Declaring our loaders and the different extensions - loader_details = [ + self.extended_loader_details = [ (ROSMsgLoader, ['.msg']), (ROSSrvLoader, ['.srv']), ] - super(ROSFileFinder, self).__init__(path, *loader_details) - - # First we need to skip all the cases that we are not concerned with - - # if path_entry != self.PATH_TRIGGER: - # self.logger.debug('ROSImportFinder does not work for %s' % path_entry) - # raise ImportError() - - # Adding the share path (where we can find all packages and their messages) - # share_path = os.path.join(workspace_entry, 'share') - # if os.path.exists(share_path): - # self.share_path = share_path - # - # python_libpath = os.path.join(workspace_entry, 'lib', - # 'python' + sys.version_info.major + '.' + sys.version_info.minor) - # if os.path.exists(python_libpath): - # # adding python site directories for the ROS distro (to find python modules as usual with ROS) - # if os.path.exists(os.path.join(python_libpath, 'dist-packages')): - # site.addsitedir(os.path.join(python_libpath, 'dist-packages')) - # if os.path.exists(os.path.join(python_libpath, 'site-packages')): - # site.addsitedir(os.path.join(python_libpath, 'site-packages')) - # # Note : here we want to keep a pure python logic to not pollute the environment (not like with pyros-setup) - # - # # self.rospkgs = [] - # # for root, dirs, files in os.walk(self.share_path, topdown=False): - # # if 'package.xml' in files: # we have found a ros package - # # - # # - # # - # # self.rospkgs.append(root) - # # found_one = True - # # if not found_one: - # # raise ImportError("No ROS package found in {0}".format(workspace_entry)) - # - # else: # this is not a workspace, maybe we are expected to get the package directly from source ? - # found_one = False - # self.src_rospkgs = [] - # for root, dirs, files in os.walk(workspace_entry, topdown=False): - # if 'package.xml' in files: # we have found a ros package - # self.src_rospkgs.append(root) - # found_one = True - # if not found_one: - # raise ImportError("No ROS package found in {0}".format(workspace_entry)) - - # # path_entry contains the path where the finder has been instantiated... - # if (not os.path.exists(path_entry) or # non existent path - # not os.path.basename(path_entry) in ['msg', 'srv'] # path not terminated with msg/srv (msg and srv inside python source) - # ): - # raise ImportError # we raise if we cannot find msg or srv folder - - # Then we can do the initialisation - self.logger = logging.getLogger(__name__) - self.logger.debug('Checking ROSFileFinder support for %s' % path) - - - def _get_spec(self, loader_class, fullname, path, smsl, target): - loader = loader_class(fullname, path) - return importlib.util.spec_from_file_location(fullname, path, loader=loader, - submodule_search_locations=smsl) + # We rely on FileFinder, just with different loaders for different file extensions + super(ROSFileFinder, self).__init__( + path, + *self.extended_loader_details + ) def find_spec(self, fullname, target=None): """ @@ -117,61 +63,35 @@ def find_spec(self, fullname, target=None): :return: """ - print('ROSFileFinder looking for "%s"' % fullname) tail_module = fullname.rpartition('.')[2] - + loader = None + spec = None # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. base_path = os.path.join(self.path, tail_module) if os.path.isdir(base_path): - found_one = False - loader = None for suffix, loader_class in self._loaders: - loader = loader_class if [f for f in os.listdir(base_path) if f.endswith(suffix)] else loader + loader = loader_class(fullname, base_path) if [f for f in os.listdir(base_path) if f.endswith(suffix)] else loader # DO we need one or two loaders ? (package logic is same, but msg or srv differs after...) - spec = self._get_spec(loader, fullname, base_path, [base_path], target) + if loader: + spec = importlib.util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path]) else: - # we use default behavior - spec = super(ROSFileFinder, self).find_spec(fullname=fullname, target=target) + if tail_module.startswith('_'): # consistent with generated module from msg package + base_path = os.path.join(self.path, tail_module[1:]) + for suffix, loader_class in self._loaders: + loader = loader_class(fullname, base_path + suffix) if os.path.isfile(base_path + suffix) else loader + if loader: + spec = importlib.util.spec_from_file_location(fullname, loader.path, loader=loader) + else: # probably an attempt to import the class in the generated module (shortcutting python import mechanism) + base_path = os.path.join(self.path, tail_module) + for suffix, loader_class in self._loaders: + loader = loader_class(fullname, base_path + suffix) if os.path.isfile(base_path + suffix) else loader + if loader: + spec = importlib.util.spec_from_file_location(fullname, loader.path, loader=loader) + + # we use default behavior if we couldn't find a spec before + spec = spec or super(ROSFileFinder, self).find_spec(fullname=fullname, target=target) return spec - # TODO: read PEP 420 :) - last_mile = fullname.split('.')[-1] - - for src_pkg in self.src_rospkgs: - if os.path.basename(src_pkg) == fullname: # we found it - return importlib.machinery.ModuleSpec( - fullname, - None, # TODO loader - origin=src_pkg, - is_package=True - ) - - # If we get here, we need to find the package in a workspace... - - # If we can find it in share/ with messages, we can generate them. - # good or bad idea ? - # Else, we can use the usual python import (since we added them to site) - #return importlib.util..(fullname). - - for root, dirs, files in os.walk(self.workspace_entry, topdown=False): - if last_mile + '.msg' in files: - return importlib.machinery.ModuleSpec( - fullname, - None, # TODO loader - origin=root, - is_package=False - ) - if last_mile + '.srv' in files: - return importlib.machinery.ModuleSpec( - fullname, - None, # TODO loader - origin=root, - is_package=False - ) - - # we couldn't find the module. let someone else find it. - return None - class ROSImportFinder(importlib.abc.PathEntryFinder): # TODO : we can use this to enable ROS Finder only on relevant paths (ros distro paths + dev workspaces) from sys.path diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index 1cd7918..353e1d8 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -5,6 +5,8 @@ import site import tempfile +import shutil + from pyros_msgs.importer import rosmsg_generator """ @@ -53,126 +55,78 @@ def _get_key_name(fullname, db): import importlib.abc import importlib.machinery - # To do only once when starting up - rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport') + # To do only once when starting up (for this process) + rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) if os.path.exists(rosimport_path): - os.removedirs(rosimport_path) + shutil.rmtree(rosimport_path) os.makedirs(rosimport_path) - class ROSMsgLoader(importlib.abc.SourceLoader): - + class ROSMsgLoader(importlib.machinery.SourceFileLoader): def __init__(self, fullname, path): self.logger = logging.getLogger(__name__) self.path = path - self.msgsrv_files = [f for f in os.listdir(self.path) if f.endswith('.msg')] - self.rospackage = fullname.partition('.')[0] - - self.outdir_pkg = tempfile.mkdtemp(prefix=self.rospackage, dir=rosimport_path) - - # TODO : we need to determine that from the loader - self.includepath = [] - self.ns_pkg = False # doesnt seem needed yet and is overall safer... - - gen = rosmsg_generator.genmsgsrv_py( - msgsrv_files=self.msgsrv_files, - package=self.rospackage, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - ns_pkg=self.ns_pkg - ) - - def get_data(self, path): - """Returns the bytes from the source code (will be used to generate bytecode)""" - self.get_filename() - return None - - - def get_source(self, fullname): - self.logger.debug('loading source for "%s" from msg' % fullname) - try: - - self._generate() - - # with shelve_context(self.path_entry) as db: - # key_name = _get_key_name(fullname, db) - # if key_name: - # return db[key_name] - # raise ImportError('could not find source for %s' % fullname) - - pass + # We should reproduce package structure in generated file structure + dirlist = self.path.split(os.sep) + pkgidx = dirlist[::-1].index(self.rospackage) + indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != 'msg' and not p.endswith('.msg')] + self.outdir_pkg = os.path.join(rosimport_path, self.rospackage, *indirlist[::-1]) + + # : hack to be able to import a generated class (if requested) + self.requested_class = None + + if os.path.isdir(self.path): + + # TODO : we need to determine that from the loader + self.includepath = [] + self.ns_pkg = False # namespace can prevent just having a 'msg' folder in a python package + + # TODO : dynamic in memory generation (we do not need the file ultimately...) + self.gen_msgs = rosmsg_generator.genmsg_py( + msg_files=[os.path.join(self.path, f) for f in os.listdir(self.path)], # every file not ending in '.msg' will be ignored + package=self.rospackage, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + initpy=True # we always create an __init__.py when called from here. + ) + init_path = None + for pyf in self.gen_msgs: + if pyf.endswith('__init__.py'): + init_path = pyf + + if not init_path: + raise ImportError("__init__.py file not found after python msg package generation") + + # relying on usual source file loader since we have generated normal python code + super(ROSMsgLoader, self).__init__(fullname, init_path) + + elif os.path.isfile(self.path): + # The file should have already been generated (by the loader for a msg package) + # Note we do not want to rely on namespace packages here, since they are not standardized for python2, + # and they can prevent some useful usecases. + + # Hack to be able to "import generated classes" + modname = fullname.rpartition('.')[2] + if modname.startswith('_'): # the init importing + filepath = os.path.join(self.outdir_pkg, 'msg', modname + '.py') + else: + self.requested_class = modname + filepath = os.path.join(self.outdir_pkg, 'msg', '_' + modname + '.py') + # relying on usual source file loader since we have previously generated normal python code + super(ROSMsgLoader, self).__init__(fullname, filepath) + + def exec_module(self, module): + super(ROSMsgLoader, self).exec_module(module=module) + + # looking for a class in the module to import if it was requested. + if self.requested_class and self.requested_class in vars(module): + # we replace the module with the actual class (the module is still referenced with '_' at the beginning + sys.modules[self.name] = getattr(module, self.requested_class) - except Exception as e: - self.logger.debug('could not load source:', e) - raise ImportError(str(e)) - - # defining this to benefit from backward compat import mechanism in python 3.X - def get_filename(self, name): - """ - Deterministic temporary filename. - :param name: - :return: - """ - os.sep.join(name.split(".")) + '.' + self.ext - - # def _get_filename(self, fullname): - # # Make up a fake filename that starts with the path entry - # # so pkgutil.get_data() works correctly. - # return os.path.join(self.path_entry, fullname) - # - # # defining this to benefit from backward compat import mechanism in python 3.X - # def is_package(self, name): - # names = name.split(".") - # parent_idx = len(names) -1 - # # trying to find a parent already loaded - # while 0<= parent_idx < len(names) : - # if names[parent_idx] in sys.modules: # we found a parent, we can get his path and go back - # pass - # else: # parent not found, need to check its parent - # parent_idx-=1 - # - # - # else: # No loaded parent found, lets attempt to import it directly (searching in all sys.paths) - # - # pass - # - # - # return None # TODO : implement check - # - # def load_module(self, fullname): - # - # if fullname in sys.modules: - # self.logger.debug('reusing existing module from previous import of "%s"' % fullname) - # mod = sys.modules[fullname] - # else: - # self.logger.debug('creating a new module object for "%s"' % fullname) - # mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) - # - # # Set a few properties required by PEP 302 - # mod.__file__ = self._get_filename(fullname) - # mod.__name__ = fullname - # mod.__path__ = self.path_entry - # mod.__loader__ = self - # mod.__package__ = '.'.join(fullname.split('.')[:-1]) - # - # if self.is_package(fullname): - # self.logger.debug('adding path for package') - # # Set __path__ for packages - # # so we can find the sub-modules. - # mod.__path__ = [self.path_entry] - # else: - # self.logger.debug('imported as regular module') - # - # source = self.get_source(fullname) - # - # self.logger.debug('execing source...') - # exec(source, mod.__dict__) - # self.logger.debug('done') - # return mod class ROSSrvLoader(importlib.abc.SourceLoader): diff --git a/tests/importer/test_rosmsg_import_msgmod_relative.py b/tests/importer/test_rosmsg_import_msgmod_relative.py index a9569be..c3c1a21 100644 --- a/tests/importer/test_rosmsg_import_msgmod_relative.py +++ b/tests/importer/test_rosmsg_import_msgmod_relative.py @@ -172,13 +172,16 @@ def test_import_absolute_module(self): import pyros_msgs.importer.tests.msg.TestMsg as msg_mod self.assertTrue(msg_mod is not None) + self.assertTrue(msg_mod._type == 'pyros_msgs/TestMsg') @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") def test_importlib_import_absolute(self): # Verify that files exists and are importable - msg_mod = importlib.__import__('pyros_msgs.importer.tests.msg.TestMsg',) + importlib.__import__('pyros_msgs.importer.tests.msg.TestMsg',) + msg_mod = sys.modules['pyros_msgs.importer.tests.msg.TestMsg'] self.assertTrue(msg_mod is not None) + self.assertTrue(msg_mod._type == 'pyros_msgs/TestMsg') @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") def test_importlib_loadmodule_absolute(self): diff --git a/tests/importer/test_rosmsg_import_msgpkg_relative.py b/tests/importer/test_rosmsg_import_msgpkg_relative.py index 03b0a14..ffc079d 100644 --- a/tests/importer/test_rosmsg_import_msgpkg_relative.py +++ b/tests/importer/test_rosmsg_import_msgpkg_relative.py @@ -175,13 +175,18 @@ def test_import_absolute_pkg(self): msg_mod = msg_pkg.TestMsg self.assertTrue(msg_mod is not None) + self.assertTrue(msg_mod._type == 'pyros_msgs/TestMsg') @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") def test_importlib_import_absolute(self): # Verify that files exists and are importable - msg_pkg = importlib.__import__('pyros_msgs.importer.tests.msg',) - + importlib.__import__('pyros_msgs.importer.tests.msg',) + msg_pkg = sys.modules['pyros_msgs.importer.tests.msg'] self.assertTrue(msg_pkg is not None) + + msg_mod = msg_pkg.TestMsg + self.assertTrue(msg_mod is not None) + self.assertTrue(msg_mod._type == 'pyros_msgs/TestMsg') @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") def test_importlib_loadmodule_absolute(self): From 57f656525d23c5af82a06c295a619fa7b38a9375 Mon Sep 17 00:00:00 2001 From: AlexV Date: Thu, 22 Jun 2017 19:03:34 +0900 Subject: [PATCH 09/28] WIP import test passing, added pytest-boxed. --- .gitmodules | 3 + pydeps/pytest-boxed | 1 + pyros_msgs/importer/rosmsg_finder.py | 100 +++++---- pyros_msgs/importer/rosmsg_loader.py | 110 +++++----- .../importer/tests/test_rosmsg_import.py | 191 ++++++++++++++++++ ...g_absolute.py => test_rosmsg_importlib.py} | 44 +--- .../test_rosmsg_import_msgpkg_relative.py | 13 +- 7 files changed, 334 insertions(+), 128 deletions(-) create mode 160000 pydeps/pytest-boxed create mode 100644 pyros_msgs/importer/tests/test_rosmsg_import.py rename pyros_msgs/importer/tests/{test_rosmsg_import_pkg_absolute.py => test_rosmsg_importlib.py} (84%) diff --git a/.gitmodules b/.gitmodules index a9868ce..a400b51 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "rosdeps/std_msgs"] path = rosdeps/std_msgs url = https://github.com/ros/std_msgs.git +[submodule "pydeps/pytest-boxed"] + path = pydeps/pytest-boxed + url = https://github.com/pytest-dev/pytest-boxed diff --git a/pydeps/pytest-boxed b/pydeps/pytest-boxed new file mode 160000 index 0000000..9cf9bcc --- /dev/null +++ b/pydeps/pytest-boxed @@ -0,0 +1 @@ +Subproject commit 9cf9bccf73ae616729c0f2b799c457d3b985e47f diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py index 26a1948..c8177e3 100644 --- a/pyros_msgs/importer/rosmsg_finder.py +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -32,28 +32,36 @@ import importlib.util - class ROSFileFinder(importlib.machinery.FileFinder): + class DirectoryFinder(importlib.machinery.FileFinder): + """Finder to interpret directories as modules, and files as classes""" - def __init__(self, path): + def __init__(self, path, *ros_loader_details): """ - Finder to get ROS specific files and directories (message and services files). - It need to be inserted in sys.path_hooks before FileFinder, since these are Files but not python ones. + Finder to get directories containing ROS message and service files. + It need to be inserted in sys.path_hooks before FileFinder, since these are Directories but not containing __init__ as per python hardcoded convention. + + Note: There is a matching issue between msg/ folder and msg/My.msg on one side, and package, module, class concepts on the other. + Since a module is not callable, but we need to call My(data) to build a message class (ROS convention), we match the msg/ folder to a module (and not a package) + And to keep matching ROS conventions, a directory without __init__ or any message/service file, will become a namespace (sub)package. :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) """ - # Declaring our loaders and the different extensions - self.extended_loader_details = [ - (ROSMsgLoader, ['.msg']), - (ROSSrvLoader, ['.srv']), - ] + ros_loaders = [] + for loader, suffixes in ros_loader_details: + ros_loaders.extend((suffix, loader) for suffix in suffixes) + self._ros_loaders = ros_loaders - # We rely on FileFinder, just with different loaders for different file extensions - super(ROSFileFinder, self).__init__( + # We rely on FileFinder and python loader to deal with our generated code + super(DirectoryFinder, self).__init__( path, - *self.extended_loader_details + (importlib.machinery.SourceFileLoader, ['.py']), + (importlib.machinery.SourcelessFileLoader, ['.pyc']), ) + def __repr__(self): + return 'DirectoryFinder({!r})'.format(self.path) + def find_spec(self, fullname, target=None): """ Try to find a spec for the specified module. @@ -64,34 +72,56 @@ def find_spec(self, fullname, target=None): """ tail_module = fullname.rpartition('.')[2] - loader = None + spec = None - # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. + + # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. + # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package + # tail_module = tail_module[1:] base_path = os.path.join(self.path, tail_module) + + # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. if os.path.isdir(base_path): - for suffix, loader_class in self._loaders: - loader = loader_class(fullname, base_path) if [f for f in os.listdir(base_path) if f.endswith(suffix)] else loader - # DO we need one or two loaders ? (package logic is same, but msg or srv differs after...) - if loader: - spec = importlib.util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path]) - else: - if tail_module.startswith('_'): # consistent with generated module from msg package - base_path = os.path.join(self.path, tail_module[1:]) - for suffix, loader_class in self._loaders: - loader = loader_class(fullname, base_path + suffix) if os.path.isfile(base_path + suffix) else loader - if loader: - spec = importlib.util.spec_from_file_location(fullname, loader.path, loader=loader) - else: # probably an attempt to import the class in the generated module (shortcutting python import mechanism) - base_path = os.path.join(self.path, tail_module) - for suffix, loader_class in self._loaders: - loader = loader_class(fullname, base_path + suffix) if os.path.isfile(base_path + suffix) else loader - if loader: - spec = importlib.util.spec_from_file_location(fullname, loader.path, loader=loader) - - # we use default behavior if we couldn't find a spec before - spec = spec or super(ROSFileFinder, self).find_spec(fullname=fullname, target=target) + loader_class = None + rosdir = None + # Figuring out if we should care about this directory at all + for root, dirs, files in os.walk(base_path): + for suffix, loader_cls in self._ros_loaders: + if any(f.endswith(suffix) for f in files): + loader_class = loader_cls + rosdir = root + if loader_class and rosdir and rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module + loader = loader_class(fullname, base_path) + # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location + spec = importlib.util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) + # We DO NOT WANT TO add the generated dir in sys.path to use a python loader + # since the plan is to eventually not have to rely on files at all TODO + + # Relying on FileFinder if we couldn't find any specific directory structure/content + # It will return a namespace spec if no file can be found + # or will return a proper loader for already generated python files + spec = spec or super(DirectoryFinder, self).find_spec(fullname, target=target) + # we return None if we couldn't find a spec before return spec + MSG_SUFFIXES = ['.msg'] + SRV_SUFFIXES = ['.srv'] + + def _get_supported_ros_loaders(): + """Returns a list of file-based module loaders. + Each item is a tuple (loader, suffixes). + """ + msg = ROSMsgLoader, MSG_SUFFIXES + srv = ROSSrvLoader, SRV_SUFFIXES + return [msg, srv] + + + def _install(): + """Install the path-based import components.""" + supported_loaders = _get_supported_ros_loaders() + sys.path_hooks.extend([DirectoryFinder.path_hook(*supported_loaders)]) + # TODO : sys.meta_path.append(DistroFinder) + class ROSImportFinder(importlib.abc.PathEntryFinder): # TODO : we can use this to enable ROS Finder only on relevant paths (ros distro paths + dev workspaces) from sys.path diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index 353e1d8..a47dc7d 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -55,76 +55,96 @@ def _get_key_name(fullname, db): import importlib.abc import importlib.machinery - # To do only once when starting up (for this process) - rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) - if os.path.exists(rosimport_path): - shutil.rmtree(rosimport_path) - os.makedirs(rosimport_path) - class ROSMsgLoader(importlib.machinery.SourceFileLoader): def __init__(self, fullname, path): self.logger = logging.getLogger(__name__) - self.path = path + + # Doing this in each loader, in case we are running from different processes, + # avoiding to reload from same file (especially useful for boxed tests). + # But deterministic path to avoid regenerating from the same interpreter + self.rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) + if os.path.exists(self.rosimport_path): + shutil.rmtree(self.rosimport_path) + os.makedirs(self.rosimport_path) self.rospackage = fullname.partition('.')[0] # We should reproduce package structure in generated file structure - dirlist = self.path.split(os.sep) + dirlist = path.split(os.sep) pkgidx = dirlist[::-1].index(self.rospackage) indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != 'msg' and not p.endswith('.msg')] - self.outdir_pkg = os.path.join(rosimport_path, self.rospackage, *indirlist[::-1]) + self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) # : hack to be able to import a generated class (if requested) - self.requested_class = None - - if os.path.isdir(self.path): - - # TODO : we need to determine that from the loader - self.includepath = [] - self.ns_pkg = False # namespace can prevent just having a 'msg' folder in a python package - - # TODO : dynamic in memory generation (we do not need the file ultimately...) - self.gen_msgs = rosmsg_generator.genmsg_py( - msg_files=[os.path.join(self.path, f) for f in os.listdir(self.path)], # every file not ending in '.msg' will be ignored - package=self.rospackage, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - initpy=True # we always create an __init__.py when called from here. - ) - init_path = None - for pyf in self.gen_msgs: - if pyf.endswith('__init__.py'): - init_path = pyf - - if not init_path: - raise ImportError("__init__.py file not found after python msg package generation") - - # relying on usual source file loader since we have generated normal python code - super(ROSMsgLoader, self).__init__(fullname, init_path) - - elif os.path.isfile(self.path): + # self.requested_class = None + + if os.path.isdir(path): + if path.endswith('msg') and any([f.endswith('.msg') for f in os.listdir(path)]): # if we get a non empty 'msg' folder + init_path = os.path.join(self.outdir_pkg, 'msg', '__init__.py') + if not os.path.exists(init_path): + # TODO : we need to determine that from the loader + # as a minimum we need to add current package + self.includepath = [self.rospackage + ':'+ path] + + # TODO : dynamic in memory generation (we do not need the file ultimately...) + self.gen_msgs = rosmsg_generator.genmsg_py( + msg_files=[os.path.join(path, f) for f in os.listdir(path)], # every file not ending in '.msg' will be ignored + package=self.rospackage, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + initpy=True # we always create an __init__.py when called from here. + ) + init_path = None + for pyf in self.gen_msgs: + if pyf.endswith('__init__.py'): + init_path = pyf + + if not init_path: + raise ImportError("__init__.py file not found".format(init_path)) + if not os.path.exists(init_path): + raise ImportError("{0} file not found".format(init_path)) + + # relying on usual source file loader since we have generated normal python code + super(ROSMsgLoader, self).__init__(fullname, init_path) + else: # it is a directory potentially containing an 'msg' + # If we are here, it means it wasn't loaded before + # We need to be able to load from source + super(ROSMsgLoader, self).__init__(fullname, path) + + # or to load from installed ros package (python already generated, no point to generate again) + # Note : the path being in sys.path or not is a matter of ROS setup or metafinder. + # TODO + + elif os.path.isfile(path): # The file should have already been generated (by the loader for a msg package) # Note we do not want to rely on namespace packages here, since they are not standardized for python2, # and they can prevent some useful usecases. # Hack to be able to "import generated classes" modname = fullname.rpartition('.')[2] - if modname.startswith('_'): # the init importing - filepath = os.path.join(self.outdir_pkg, 'msg', modname + '.py') - else: - self.requested_class = modname - filepath = os.path.join(self.outdir_pkg, 'msg', '_' + modname + '.py') + filepath = os.path.join(self.outdir_pkg, 'msg', '_' + modname + '.py') # the generated module # relying on usual source file loader since we have previously generated normal python code super(ROSMsgLoader, self).__init__(fullname, filepath) + def get_gen_path(self): + """Returning the generated path matching the import""" + return os.path.join(self.outdir_pkg, 'msg') + def exec_module(self, module): super(ROSMsgLoader, self).exec_module(module=module) + # bypassing the __init__ stage + #if self.name in module + + + if self.name.endswith('.msg'): # we are talking about the package + pass + # looking for a class in the module to import if it was requested. - if self.requested_class and self.requested_class in vars(module): - # we replace the module with the actual class (the module is still referenced with '_' at the beginning - sys.modules[self.name] = getattr(module, self.requested_class) + # if self.requested_class and self.requested_class in vars(module): + # # we replace the module with the actual class (the module is still referenced with '_' at the beginning + # sys.modules[self.name] = getattr(module, self.requested_class) diff --git a/pyros_msgs/importer/tests/test_rosmsg_import.py b/pyros_msgs/importer/tests/test_rosmsg_import.py new file mode 100644 index 0000000..7a89455 --- /dev/null +++ b/pyros_msgs/importer/tests/test_rosmsg_import.py @@ -0,0 +1,191 @@ +from __future__ import absolute_import, division, print_function + +import copy + +""" +Testing rosmsg_import with import keyword. +CAREFUL : these tests should run with pytest --boxed in order to avoid polluting each other sys.modules +""" + +import os +import sys +import runpy +import logging.config + +logging.config.dictConfig({ + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'default': { + 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + } + }, + 'root': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, +}) + +# Since test frameworks (like pytest) play with the import machinery, we cannot use it here... +import unittest + +# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path + +import importlib + +# Importing importer module +from pyros_msgs.importer import rosmsg_finder + +# importlib +# https://pymotw.com/3/importlib/index.html +# https://pymotw.com/2/importlib/index.html + + +# HACK to fix spec from pytest +# @property +# def spec(): +# importlib.util.spec_from_file_location(__file__) + + +def print_importers(): + import sys + import pprint + + print('PATH:'), + pprint.pprint(sys.path) + print() + print('IMPORTERS:') + for name, cache_value in sys.path_importer_cache.items(): + name = name.replace(sys.prefix, '...') + print('%s: %r' % (name, cache_value)) + + +class TestImportAnotherMsg(unittest.TestCase): + + rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') + + @classmethod + def setUpClass(cls): + # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package + supported_loaders = rosmsg_finder._get_supported_ros_loaders() + ros_hook = rosmsg_finder.DirectoryFinder.path_hook(*supported_loaders) + sys.path_hooks.insert(1, ros_hook) + + sys.path.append(cls.rosdeps_path) + + @classmethod + def tearDownClass(cls): + # CAREFUL : Even though we remove the path from sys.path, + # initialized finders will remain in sys.path_importer_cache + sys.path.remove(cls.rosdeps_path) + + def test_import_absolute_pkg(self): + print_importers() + + # Verify that files exists and are importable + import std_msgs.msg as std_msgs + + self.assertTrue(std_msgs is not None) + self.assertTrue(std_msgs.Bool is not None) + self.assertTrue(callable(std_msgs.Bool)) + self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') + + def test_import_class_from_absolute_pkg(self): + """Verify that""" + print_importers() + + # Verify that files exists and are importable + from std_msgs.msg import Bool + + self.assertTrue(Bool is not None) + self.assertTrue(callable(Bool)) + self.assertTrue(Bool._type == 'std_msgs/Bool') + + def test_import_relative_pkg(self): + """Verify that package is importable relatively""" + print_importers() + + from . import msg as test_msgs + + self.assertTrue(test_msgs is not None) + self.assertTrue(test_msgs.TestMsg is not None) + self.assertTrue(callable(test_msgs.TestMsg)) + self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name + + def test_import_class_from_relative_pkg(self): + """Verify that message class is importable relatively""" + print_importers() + + from .msg import TestMsg + + self.assertTrue(TestMsg is not None) + self.assertTrue(callable(TestMsg)) + self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') + + def test_import_class_absolute_raises(self): + print_importers() + + with self.assertRaises(ImportError): + import std_msgs.msg.Bool + + # TODO + # def test_double_import_uses_cache(self): + # # cleaning previously imported module + # if 'std_msgs' in sys.modules: + # sys.modules.pop('std_msgs') + # + # print_importers() + # # Verify that files exists and are importable + # import std_msgs.msg as std_msgs + # + # self.assertTrue(std_msgs.Bool is not None) + # self.assertTrue(callable(std_msgs.Bool)) + # self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') + # + # import std_msgs.msg as std_msgs2 + # + # self.assertTrue(std_msgs == std_msgs2) + + + + + +def test_importlib_srv_module(): + pass + # TODO + # # Verify that files exists and are importable + # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') + + +# imp https://pymotw.com/2/imp/index.html +# TODO + +# def test_imp_msg_module(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.msg.TestMsg') +# +# +# def test_imp_msg_pkg(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.msg') +# +# +# def test_imp_srv_module(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.srv.TestSrv') +# +# +# def test_imp_srv_pkg(): +# # Verify that files exists and are importable +# msg_mod = imp.import_module('test_gen_msgs.srv') + + +if __name__ == '__main__': + unittest.main() + #import nose + #nose.main() diff --git a/pyros_msgs/importer/tests/test_rosmsg_import_pkg_absolute.py b/pyros_msgs/importer/tests/test_rosmsg_importlib.py similarity index 84% rename from pyros_msgs/importer/tests/test_rosmsg_import_pkg_absolute.py rename to pyros_msgs/importer/tests/test_rosmsg_importlib.py index bac8b05..c4dd594 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_import_pkg_absolute.py +++ b/pyros_msgs/importer/tests/test_rosmsg_importlib.py @@ -150,7 +150,7 @@ def print_importers(): print('%s: %r' % (name, cache_value)) -class TestImportAnotherMsg(unittest.TestCase): +class TestImportLibAnotherMsg(unittest.TestCase): rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') @@ -158,7 +158,8 @@ class TestImportAnotherMsg(unittest.TestCase): def setUpClass(cls): # We need to be before FileFinder to be able to find our (non .py[c]) files # inside, maybe already imported, python packages... - sys.path_hooks.insert(1, rosmsg_finder.ROSImportFinder) + sys.path_hooks.insert(1, rosmsg_finder.ROSFileFinder) + #sys.path_hooks.append(rosmsg_finder.ROSFileFinder) sys.path.append(cls.rosdeps_path) @classmethod @@ -167,45 +168,10 @@ def tearDownClass(cls): # initialized finders will remain in sys.path_importer_cache sys.path.remove(cls.rosdeps_path) - def test_import_absolute(self): - print_importers() - # Verify that files exists and are importable - import std_msgs.msg.Bool as msg_bool - - self.assertTrue(msg_bool is not None) - - def test_import_from(self): - - print_importers() - # Verify that files exists and are importable - try: - # Using std_msgs directly if ROS has been setup (while using from ROS pkg) - from std_msgs.msg import Bool as msg_bool - except ImportError: - # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) - import site - site.addsitedir(os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), - 'ros-site' - )) - from std_msgs.msg import Bool as msg_bool - - self.assertTrue(msg_bool is not None) - @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_absolute(self): + def test_importlib_import_absolute_pkg(self): # Verify that files exists and are importable - try: - # Using std_msgs directly if ROS has been setup (while using from ROS pkg) - msg_bool = importlib.__import__('std_msgs.msg.Bool', ) - except ImportError: - # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) - import site - site.addsitedir(os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), - 'ros-site' - )) - msg_bool = importlib.__import__('std_msgs.msg.Bool',) + importlib.__import__('std_msgs.msg.Bool') assert msg_bool is not None diff --git a/tests/importer/test_rosmsg_import_msgpkg_relative.py b/tests/importer/test_rosmsg_import_msgpkg_relative.py index ffc079d..1bd0d47 100644 --- a/tests/importer/test_rosmsg_import_msgpkg_relative.py +++ b/tests/importer/test_rosmsg_import_msgpkg_relative.py @@ -169,13 +169,10 @@ def tearDownClass(cls): def test_import_absolute_pkg(self): print_importers() # Verify that files exists and are importable - # Note : this requires a metapath finder import pyros_msgs.importer.tests.msg as msg_pkg self.assertTrue(msg_pkg is not None) - - msg_mod = msg_pkg.TestMsg - self.assertTrue(msg_mod is not None) - self.assertTrue(msg_mod._type == 'pyros_msgs/TestMsg') + self.assertTrue(msg_pkg.TestMsg is not None) + self.assertTrue(msg_pkg.TestMsg._type == 'pyros_msgs/TestMsg') @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") def test_importlib_import_absolute(self): @@ -183,10 +180,8 @@ def test_importlib_import_absolute(self): importlib.__import__('pyros_msgs.importer.tests.msg',) msg_pkg = sys.modules['pyros_msgs.importer.tests.msg'] self.assertTrue(msg_pkg is not None) - - msg_mod = msg_pkg.TestMsg - self.assertTrue(msg_mod is not None) - self.assertTrue(msg_mod._type == 'pyros_msgs/TestMsg') + self.assertTrue(msg_pkg.TestMsg is not None) + self.assertTrue(msg_pkg.TestMsg._type == 'pyros_msgs/TestMsg') @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") def test_importlib_loadmodule_absolute(self): From 71c21ae0ed456f1a92b887e0ab5fa22ce5726d6f Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 23 Jun 2017 12:33:38 +0900 Subject: [PATCH 10/28] importlib tests passing --- pyros_msgs/importer/rosmsg_loader.py | 13 - .../importer/tests/test_rosmsg_import.py | 81 +++--- .../importer/tests/test_rosmsg_importlib.py | 257 ++++++++++++++---- setup.py | 1 + 4 files changed, 233 insertions(+), 119 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index a47dc7d..d9eaf77 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -134,19 +134,6 @@ def get_gen_path(self): def exec_module(self, module): super(ROSMsgLoader, self).exec_module(module=module) - # bypassing the __init__ stage - #if self.name in module - - - if self.name.endswith('.msg'): # we are talking about the package - pass - - # looking for a class in the module to import if it was requested. - # if self.requested_class and self.requested_class in vars(module): - # # we replace the module with the actual class (the module is still referenced with '_' at the beginning - # sys.modules[self.name] = getattr(module, self.requested_class) - - class ROSSrvLoader(importlib.abc.SourceLoader): diff --git a/pyros_msgs/importer/tests/test_rosmsg_import.py b/pyros_msgs/importer/tests/test_rosmsg_import.py index 7a89455..45883cc 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_import.py @@ -65,7 +65,7 @@ def print_importers(): print('%s: %r' % (name, cache_value)) -class TestImportAnotherMsg(unittest.TestCase): +class TestImportMsg(unittest.TestCase): rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') @@ -84,7 +84,7 @@ def tearDownClass(cls): # initialized finders will remain in sys.path_importer_cache sys.path.remove(cls.rosdeps_path) - def test_import_absolute_pkg(self): + def test_import_absolute_msg(self): print_importers() # Verify that files exists and are importable @@ -95,7 +95,10 @@ def test_import_absolute_pkg(self): self.assertTrue(callable(std_msgs.Bool)) self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - def test_import_class_from_absolute_pkg(self): + # use it ! + self.assertTrue(std_msgs.Bool(True)) + + def test_import_class_from_absolute_msg(self): """Verify that""" print_importers() @@ -106,7 +109,10 @@ def test_import_class_from_absolute_pkg(self): self.assertTrue(callable(Bool)) self.assertTrue(Bool._type == 'std_msgs/Bool') - def test_import_relative_pkg(self): + # use it ! + self.assertTrue(Bool(True)) + + def test_import_relative_msg(self): """Verify that package is importable relatively""" print_importers() @@ -117,7 +123,10 @@ def test_import_relative_pkg(self): self.assertTrue(callable(test_msgs.TestMsg)) self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name - def test_import_class_from_relative_pkg(self): + # use it ! + self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) + + def test_import_class_from_relative_msg(self): """Verify that message class is importable relatively""" print_importers() @@ -127,29 +136,27 @@ def test_import_class_from_relative_pkg(self): self.assertTrue(callable(TestMsg)) self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') - def test_import_class_absolute_raises(self): + # use it ! + self.assertTrue(TestMsg(test_bool=True, test_string='Test').test_bool) + + def test_import_absolute_class_raises(self): print_importers() with self.assertRaises(ImportError): import std_msgs.msg.Bool - # TODO - # def test_double_import_uses_cache(self): - # # cleaning previously imported module - # if 'std_msgs' in sys.modules: - # sys.modules.pop('std_msgs') - # - # print_importers() - # # Verify that files exists and are importable - # import std_msgs.msg as std_msgs - # - # self.assertTrue(std_msgs.Bool is not None) - # self.assertTrue(callable(std_msgs.Bool)) - # self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - # - # import std_msgs.msg as std_msgs2 - # - # self.assertTrue(std_msgs == std_msgs2) + def test_double_import_uses_cache(self): # + print_importers() + # Verify that files exists and are importable + import std_msgs.msg as std_msgs + + self.assertTrue(std_msgs.Bool is not None) + self.assertTrue(callable(std_msgs.Bool)) + self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') + + import std_msgs.msg as std_msgs2 + + self.assertTrue(std_msgs == std_msgs2) @@ -162,30 +169,6 @@ def test_importlib_srv_module(): # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') -# imp https://pymotw.com/2/imp/index.html -# TODO - -# def test_imp_msg_module(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.msg.TestMsg') -# -# -# def test_imp_msg_pkg(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.msg') -# -# -# def test_imp_srv_module(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.srv.TestSrv') -# -# -# def test_imp_srv_pkg(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.srv') - - if __name__ == '__main__': - unittest.main() - #import nose - #nose.main() + import pytest + pytest.main(['-s', '-x', __file__, '--boxed']) diff --git a/pyros_msgs/importer/tests/test_rosmsg_importlib.py b/pyros_msgs/importer/tests/test_rosmsg_importlib.py index c4dd594..bcf24c0 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_importlib.py +++ b/pyros_msgs/importer/tests/test_rosmsg_importlib.py @@ -151,15 +151,15 @@ def print_importers(): class TestImportLibAnotherMsg(unittest.TestCase): - rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') @classmethod def setUpClass(cls): - # We need to be before FileFinder to be able to find our (non .py[c]) files - # inside, maybe already imported, python packages... - sys.path_hooks.insert(1, rosmsg_finder.ROSFileFinder) - #sys.path_hooks.append(rosmsg_finder.ROSFileFinder) + # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package + supported_loaders = rosmsg_finder._get_supported_ros_loaders() + ros_hook = rosmsg_finder.DirectoryFinder.path_hook(*supported_loaders) + sys.path_hooks.insert(1, ros_hook) + sys.path.append(cls.rosdeps_path) @classmethod @@ -169,15 +169,83 @@ def tearDownClass(cls): sys.path.remove(cls.rosdeps_path) @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_absolute_pkg(self): + def test_importlib_import_absolute_msg(self): # Verify that files exists and are importable - importlib.__import__('std_msgs.msg.Bool') + std_msgs = importlib.__import__('std_msgs.msg') + std_msgs = std_msgs.msg + + self.assertTrue(std_msgs is not None) + self.assertTrue(std_msgs.Bool is not None) + self.assertTrue(callable(std_msgs.Bool)) + self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - assert msg_bool is not None + # use it ! + self.assertTrue(std_msgs.Bool(True)) + + @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + def test_importlib_import_absolute_class_raises(self): + with self.assertRaises(ImportError): + importlib.__import__('std_msgs.msg.Bool') + + # BROKEN 3.4 + # @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + # def test_importlib_import_relative_pkg(self): + # # Verify that files exists and are importable + # test_msgs = importlib.__import__('.msg') + # + # self.assertTrue(test_msgs is not None) + # self.assertTrue(test_msgs.TestMsg is not None) + # self.assertTrue(callable(test_msgs.TestMsg)) + # self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name + # + # # use it ! + # self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) + + # BROKEN 3.4 + # @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + # def test_importlib_import_relative_mod(self): + # # Verify that files exists and are importable + # msg = importlib.__import__('.msg.TestMsg') + # TestMsg = msg.TestMsg + # + # self.assertTrue(TestMsg is not None) + # self.assertTrue(callable(TestMsg)) + # self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name + # + # # use it ! + # self.assertTrue(TestMsg(test_bool=True, test_string='Test').test_bool) @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") - def test_importlib_loadmodule_absolute(self): + def test_importlib_loadmodule_absolute_msg(self): + # Verify that files exists and are dynamically importable + pkg_list = 'std_msgs.msg'.split('.')[:-1] + mod_list = 'std_msgs.msg'.split('.')[1:] + pkg = None + for pkg_name, mod_name in zip(pkg_list, mod_list): + pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) + pkg = pkg_loader.load_module(mod_name) + + std_msgs = pkg + + self.assertTrue(std_msgs is not None) + self.assertTrue(std_msgs.Bool is not None) + self.assertTrue(callable(std_msgs.Bool)) + self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') + + # use it ! + self.assertTrue(std_msgs.Bool(True)) + + # TODO : implement some differences and check we get them... + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(std_msgs) + else: + pass + + @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), + reason="importlib does not have attribute find_loader or load_module") + def test_importlib_loadmodule_absolute_class(self): # Verify that files exists and are dynamically importable pkg_list = 'std_msgs.msg.Bool'.split('.')[:-1] mod_list = 'std_msgs.msg.Bool'.split('.')[1:] @@ -186,47 +254,146 @@ def test_importlib_loadmodule_absolute(self): pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) pkg = pkg_loader.load_module(mod_name) - msg_mod = pkg + Bool = pkg + + self.assertTrue(Bool is not None) + self.assertTrue(callable(Bool)) + self.assertTrue(Bool._type == 'std_msgs/Bool') + + # use it ! + self.assertTrue(Bool(True)) + + # TODO : implement some differences and check we get them... + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(Bool) + else: + pass + + @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), + reason="importlib does not have attribute find_loader or load_module") + def test_importlib_loadmodule_relative_msg(self): + # Verify that files exists and are dynamically importable + pkg_list = '.msg'.split('.')[:-1] + mod_list = '.msg'.split('.')[1:] + pkg = None + for pkg_name, mod_name in zip(pkg_list, mod_list): + pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) + pkg = pkg_loader.load_module(mod_name) + + test_msgs = pkg + + self.assertTrue(test_msgs is not None) + self.assertTrue(test_msgs.TestMsg is not None) + self.assertTrue(callable(test_msgs.TestMsg)) + self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name + + # use it ! + self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) + + # TODO : implement some differences and check we get them... + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(test_msgs) + else: + pass + + @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), + reason="importlib does not have attribute find_loader or load_module") + def test_importlib_loadmodule_relative_class(self): + # Verify that files exists and are dynamically importable + pkg_list = '.msg.TestMsg'.split('.')[:-1] + mod_list = '.msg.TestMsg'.split('.')[1:] + pkg = None + for pkg_name, mod_name in zip(pkg_list, mod_list): + pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) + pkg = pkg_loader.load_module(mod_name) + + TestMsg = pkg + + self.assertTrue(TestMsg is not None) + self.assertTrue(callable(TestMsg)) + self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') + # use it ! + self.assertTrue(TestMsg(test_bool=True, test_string='Test').test_bool) + + # TODO : implement some differences and check we get them... if hasattr(importlib, 'reload'): # recent version of importlib # attempting to reload - importlib.reload(msg_mod) + importlib.reload(TestMsg) else: pass # TODO : dynamic using module_spec (python 3.5) @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") - def test_importlib_importmodule_absolute(self): + def test_importlib_importmodule_absolute_msg(self): # Verify that files exists and are dynamically importable - msg_bool = importlib.import_module('std_msgs.msg.Bool') + std_msgs = importlib.import_module('std_msgs.msg') + + self.assertTrue(std_msgs is not None) + self.assertTrue(std_msgs.Bool is not None) + self.assertTrue(callable(std_msgs.Bool)) + self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') + + # use it ! + self.assertTrue(std_msgs.Bool(True)) if hasattr(importlib, 'reload'): # recent version of importlib # attempting to reload - importlib.reload(msg_bool) + importlib.reload(std_msgs) else: pass - assert msg_bool is not None + assert std_msgs is not None + @unittest.skipIf(not hasattr(importlib, 'import_module'), + reason="importlib does not have attribute import_module") + def test_importlib_importmodule_absolute_class_raises(self): + with self.assertRaises(ImportError): + importlib.import_module('std_msgs.msg.Bool') + @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") + def test_importlib_importmodule_relative_msg(self): + # Verify that files exists and are dynamically importable + test_msgs = importlib.import_module('.msg', package=__package__) -# def test_importlib_msg_module_relative(): -# # Verify that files exists and are importable -# # TODO -# # if hasattr(importlib, 'find_loader'): # recent version of import lib -# # -# # pkg_loader = importlib.find_loader('test_gen_msgs') -# # pkg = pkg_loader.load_module() -# # -# # subpkg_loader = importlib.find_loader('msg', pkg.__path__) -# # subpkg = subpkg_loader.load_module() -# # -# # loader = importlib.find_loader('TestMsg', subpkg.__path__) -# # m = subpkg_loader.load_module() -# # -# # else: # old version -# msg_mod = importlib.import_module('.msg.TestMsg', package=__package__) + self.assertTrue(test_msgs is not None) + self.assertTrue(test_msgs.TestMsg is not None) + self.assertTrue(callable(test_msgs.TestMsg)) + self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name + + # use it ! + self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) + + if hasattr(importlib, 'reload'): # recent version of importlib + # attempting to reload + importlib.reload(test_msgs) + else: + pass + + assert test_msgs is not None + + @unittest.skipIf(not hasattr(importlib, 'import_module'), + reason="importlib does not have attribute import_module") + def test_importlib_importmodule_relative_class_raises(self): + with self.assertRaises(ImportError): + importlib.import_module('.msg.TestMsg', package=__package__) + + # TODO + # def test_double_import_uses_cache(self): # + # print_importers() + # # Verify that files exists and are importable + # import std_msgs.msg as std_msgs + # + # self.assertTrue(std_msgs.Bool is not None) + # self.assertTrue(callable(std_msgs.Bool)) + # self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') + # + # import std_msgs.msg as std_msgs2 + # + # self.assertTrue(std_msgs == std_msgs2) @@ -238,30 +405,6 @@ def test_importlib_srv_module(): # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') -# imp https://pymotw.com/2/imp/index.html -# TODO - -# def test_imp_msg_module(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.msg.TestMsg') -# -# -# def test_imp_msg_pkg(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.msg') -# -# -# def test_imp_srv_module(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.srv.TestSrv') -# -# -# def test_imp_srv_pkg(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.srv') - - if __name__ == '__main__': - unittest.main() - #import nose - #nose.main() + import pytest + pytest.main(['-s', '-x', __file__, '--boxed']) diff --git a/setup.py b/setup.py index 609660c..ad61a1d 100644 --- a/setup.py +++ b/setup.py @@ -250,6 +250,7 @@ def run(self): # 'pyros_utils', # this must be satisfied by the ROS package system... 'pyyaml>=3.10', # genpy relies on this... 'pytest>=2.8.0', # as per hypothesis requirement (careful with 2.5.1 on trusty) + 'pytest-xdist', # for --boxed (careful with the version it will be moved out of xdist) 'hypothesis>=3.0.1', # to target xenial LTS version 'numpy>=1.8.2', # from trusty version ], From 2eaa2e01e1ff110c28487a3bae1a591be51eadc6 Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 23 Jun 2017 14:39:04 +0900 Subject: [PATCH 11/28] importing services definitions now works on py3.4 --- .gitmodules | 3 + pyros_msgs/importer/rosmsg_loader.py | 98 ++++--- .../importer/tests/test_rosmsg_import.py | 132 ++++++++- rosdeps/ros_comm_msgs | 1 + .../test_rosmsg_import_msgmod_relative.py | 275 ------------------ .../test_rosmsg_import_msgpkg_relative.py | 275 ------------------ 6 files changed, 195 insertions(+), 589 deletions(-) create mode 160000 rosdeps/ros_comm_msgs delete mode 100644 tests/importer/test_rosmsg_import_msgmod_relative.py delete mode 100644 tests/importer/test_rosmsg_import_msgpkg_relative.py diff --git a/.gitmodules b/.gitmodules index a400b51..fdd19ae 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "pydeps/pytest-boxed"] path = pydeps/pytest-boxed url = https://github.com/pytest-dev/pytest-boxed +[submodule "rosdeps/ros_comm_msgs"] + path = rosdeps/ros_comm_msgs + url = https://github.com/ros/ros_comm_msgs.git diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index d9eaf77..bcd62ae 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -135,54 +135,84 @@ def exec_module(self, module): super(ROSMsgLoader, self).exec_module(module=module) - class ROSSrvLoader(importlib.abc.SourceLoader): + class ROSSrvLoader(importlib.machinery.SourceFileLoader): - def __init__(self, path_entry, msgsrv_files, package, outdir_pkg, includepath=None, ns_pkg=None): + def __init__(self, fullname, path): self.logger = logging.getLogger(__name__) - self.path_entry = path_entry - self.msgsrv_files = msgsrv_files - self.package = package - self.outdir_pkg = outdir_pkg + # Doing this in each loader, in case we are running from different processes, + # avoiding to reload from same file (especially useful for boxed tests). + # But deterministic path to avoid regenerating from the same interpreter + self.rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) + if os.path.exists(self.rosimport_path): + shutil.rmtree(self.rosimport_path) + os.makedirs(self.rosimport_path) - # TODO : we need to determine that from the loader - self.includepath = includepath - self.ns_pkg = ns_pkg + self.rospackage = fullname.partition('.')[0] + # We should reproduce package structure in generated file structure + dirlist = path.split(os.sep) + pkgidx = dirlist[::-1].index(self.rospackage) + indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != 'srv' and not p.endswith('.srv')] + self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) - def _generate(self, fullname): + # : hack to be able to import a generated class (if requested) + # self.requested_class = None - gen = rosmsg_generator.genmsgsrv_py( - msgsrv_files=self.msgsrv_files, - package=self.package, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - ns_pkg=self.ns_pkg - ) - return gen + if os.path.isdir(path): + if path.endswith('srv') and any([f.endswith('.srv') for f in os.listdir(path)]): # if we get a non empty 'msg' folder + init_path = os.path.join(self.outdir_pkg, 'srv', '__init__.py') + if not os.path.exists(init_path): + # TODO : we need to determine that from the loader + # as a minimum we need to add current package + self.includepath = [self.rospackage + ':'+ path] - def get_source(self, fullname): - self.logger.debug('loading source for "%s" from msg' % fullname) - try: + # TODO : dynamic in memory generation (we do not need the file ultimately...) + self.gen_srvs = rosmsg_generator.gensrv_py( + srv_files=[os.path.join(path, f) for f in os.listdir(path)], # every file not ending in '.msg' will be ignored + package=self.rospackage, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + initpy=True # we always create an __init__.py when called from here. + ) + init_path = None + for pyf in self.gen_srvs: + if pyf.endswith('__init__.py'): + init_path = pyf - self._generate() + if not init_path: + raise ImportError("__init__.py file not found".format(init_path)) + if not os.path.exists(init_path): + raise ImportError("{0} file not found".format(init_path)) - # with shelve_context(self.path_entry) as db: - # key_name = _get_key_name(fullname, db) - # if key_name: - # return db[key_name] - # raise ImportError('could not find source for %s' % fullname) + # relying on usual source file loader since we have generated normal python code + super(ROSSrvLoader, self).__init__(fullname, init_path) + else: # it is a directory potentially containing an 'msg' + # If we are here, it means it wasn't loaded before + # We need to be able to load from source + super(ROSSrvLoader, self).__init__(fullname, path) - pass + # or to load from installed ros package (python already generated, no point to generate again) + # Note : the path being in sys.path or not is a matter of ROS setup or metafinder. + # TODO + elif os.path.isfile(path): + # The file should have already been generated (by the loader for a msg package) + # Note we do not want to rely on namespace packages here, since they are not standardized for python2, + # and they can prevent some useful usecases. - except Exception as e: - self.logger.debug('could not load source:', e) - raise ImportError(str(e)) + # Hack to be able to "import generated classes" + modname = fullname.rpartition('.')[2] + filepath = os.path.join(self.outdir_pkg, 'srv', '_' + modname + '.py') # the generated module + # relying on usual source file loader since we have previously generated normal python code + super(ROSSrvLoader, self).__init__(fullname, filepath) - # defining this to benefit from backward compat import mechanism in python 3.X - def get_filename(self, name): - os.sep.join(name.split(".")) + '.' + self.ext + def get_gen_path(self): + """Returning the generated path matching the import""" + return os.path.join(self.outdir_pkg, 'srv') + + def exec_module(self, module): + super(ROSSrvLoader, self).exec_module(module=module) diff --git a/pyros_msgs/importer/tests/test_rosmsg_import.py b/pyros_msgs/importer/tests/test_rosmsg_import.py index 45883cc..27cac8d 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_import.py @@ -159,15 +159,137 @@ def test_double_import_uses_cache(self): # self.assertTrue(std_msgs == std_msgs2) +class TestImportSrv(unittest.TestCase): + rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps', 'ros_comm_msgs') + @classmethod + def setUpClass(cls): + # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package + supported_loaders = rosmsg_finder._get_supported_ros_loaders() + ros_hook = rosmsg_finder.DirectoryFinder.path_hook(*supported_loaders) + sys.path_hooks.insert(1, ros_hook) + + sys.path.append(cls.rosdeps_path) + + @classmethod + def tearDownClass(cls): + # CAREFUL : Even though we remove the path from sys.path, + # initialized finders will remain in sys.path_importer_cache + sys.path.remove(cls.rosdeps_path) + + def test_import_absolute_srv(self): + print_importers() + + # Verify that files exists and are importable + import std_srvs.srv as std_srvs + + self.assertTrue(std_srvs is not None) + self.assertTrue(std_srvs.SetBool is not None) + self.assertTrue(callable(std_srvs.SetBool)) + self.assertTrue(std_srvs.SetBool._type == 'std_srvs/SetBool') + + self.assertTrue(std_srvs is not None) + self.assertTrue(std_srvs.SetBoolRequest is not None) + self.assertTrue(callable(std_srvs.SetBoolRequest)) + self.assertTrue(std_srvs.SetBoolRequest._type == 'std_srvs/SetBoolRequest') + + self.assertTrue(std_srvs is not None) + self.assertTrue(std_srvs.SetBoolResponse is not None) + self.assertTrue(callable(std_srvs.SetBoolResponse)) + self.assertTrue(std_srvs.SetBoolResponse._type == 'std_srvs/SetBoolResponse') + + # use it ! + self.assertTrue(std_srvs.SetBoolRequest(data=True).data) + self.assertTrue(std_srvs.SetBoolResponse(success=True, message='Test').success) + + def test_import_class_from_absolute_srv(self): + """Verify that""" + print_importers() + + # Verify that files exists and are importable + from std_srvs.srv import SetBool, SetBoolRequest, SetBoolResponse + + self.assertTrue(SetBool is not None) + self.assertTrue(callable(SetBool)) + self.assertTrue(SetBool._type == 'std_srvs/SetBool') + + self.assertTrue(SetBoolRequest is not None) + self.assertTrue(callable(SetBoolRequest)) + self.assertTrue(SetBoolRequest._type == 'std_srvs/SetBoolRequest') + + self.assertTrue(SetBoolResponse is not None) + self.assertTrue(callable(SetBoolResponse)) + self.assertTrue(SetBoolResponse._type == 'std_srvs/SetBoolResponse') + + # use it ! + self.assertTrue(SetBoolRequest(data=True).data) + self.assertTrue(SetBoolResponse(success=True, message='Test').success) + + def test_import_relative_srv(self): + """Verify that package is importable relatively""" + print_importers() + + from . import srv as test_srvs + + self.assertTrue(test_srvs is not None) + + self.assertTrue(test_srvs.TestSrv is not None) + self.assertTrue(callable(test_srvs.TestSrv)) + self.assertTrue(test_srvs.TestSrv._type == 'pyros_msgs/TestSrv') # careful between ros package name and python package name + + self.assertTrue(test_srvs.TestSrvRequest is not None) + self.assertTrue(callable(test_srvs.TestSrvRequest)) + self.assertTrue(test_srvs.TestSrvRequest._type == 'pyros_msgs/TestSrvRequest') # careful between ros package name and python package name + + self.assertTrue(test_srvs.TestSrvResponse is not None) + self.assertTrue(callable(test_srvs.TestSrvResponse)) + self.assertTrue(test_srvs.TestSrvResponse._type == 'pyros_msgs/TestSrvResponse') # careful between ros package name and python package name + + # use it ! + self.assertTrue(test_srvs.TestSrvRequest(test_request='Test').test_request) + self.assertTrue(test_srvs.TestSrvResponse(test_response=True).test_response) + + def test_import_class_from_relative_srv(self): + """Verify that message class is importable relatively""" + print_importers() + + from .srv import TestSrv, TestSrvRequest, TestSrvResponse + + self.assertTrue(TestSrv is not None) + self.assertTrue(callable(TestSrv)) + self.assertTrue(TestSrv._type == 'pyros_msgs/TestSrv') # careful between ros package name and python package name + + self.assertTrue(TestSrvRequest is not None) + self.assertTrue(callable(TestSrvRequest)) + self.assertTrue(TestSrvRequest._type == 'pyros_msgs/TestSrvRequest') + + self.assertTrue(TestSrvResponse is not None) + self.assertTrue(callable(TestSrvResponse)) + self.assertTrue(TestSrvResponse._type == 'pyros_msgs/TestSrvResponse') + + # use it ! + self.assertTrue(TestSrvRequest(test_request='Test').test_request) + self.assertTrue(TestSrvResponse(test_response=True).test_response) + + def test_import_absolute_class_raises(self): + print_importers() + + with self.assertRaises(ImportError): + import std_srvs.srv.SetBool + + def test_double_import_uses_cache(self): # + print_importers() + # Verify that files exists and are importable + import std_srvs.srv as std_srvs + + self.assertTrue(std_srvs.SetBool is not None) + self.assertTrue(std_srvs.SetBoolRequest is not None) + self.assertTrue(std_srvs.SetBoolResponse is not None) -def test_importlib_srv_module(): - pass - # TODO - # # Verify that files exists and are importable - # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') + import std_srvs.srv as std_srvs2 + self.assertTrue(std_srvs == std_srvs2) if __name__ == '__main__': import pytest diff --git a/rosdeps/ros_comm_msgs b/rosdeps/ros_comm_msgs new file mode 160000 index 0000000..bfb8533 --- /dev/null +++ b/rosdeps/ros_comm_msgs @@ -0,0 +1 @@ +Subproject commit bfb8533fd6c6e959c71f0d2a9669baeff2dac1ad diff --git a/tests/importer/test_rosmsg_import_msgmod_relative.py b/tests/importer/test_rosmsg_import_msgmod_relative.py deleted file mode 100644 index c3c1a21..0000000 --- a/tests/importer/test_rosmsg_import_msgmod_relative.py +++ /dev/null @@ -1,275 +0,0 @@ -from __future__ import absolute_import, division, print_function -""" -Testing executing rosmsg_import outside of the package, otherwise the package is already imported. -""" - -import os -import sys -import runpy -import logging.config - -logging.config.dictConfig({ - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - } - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, -}) - -# Since test frameworks (like pytest) play with the import machinery, we cannot use it here... -import unittest - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path - -import importlib - -# Importing importer module -from pyros_msgs.importer import rosmsg_finder - -# importlib -# https://pymotw.com/3/importlib/index.html -# https://pymotw.com/2/importlib/index.html - -# if sys.version_info > (3, 4): -# import importlib.util -# # REF : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module -# # Useful to display details when debugging... -# def import_module(name, package=None): -# """An approximate implementation of import.""" -# absolute_name = importlib.util.resolve_name(name, package) -# try: -# return sys.modules[absolute_name] -# except KeyError: -# pass -# -# path = None -# if '.' in absolute_name: -# parent_name, _, child_name = absolute_name.rpartition('.') -# parent_module = import_module(parent_name) -# path = parent_module.spec.submodule_search_locations # this currently breaks (probably because of pytest custom importer ? ) -# for finder in sys.meta_path: -# spec = finder.find_spec(absolute_name, path) -# if spec is not None: -# break -# else: -# raise ImportError('No module named {absolute_name!r}'.format(**locals())) -# module = importlib.util.module_from_spec(spec) -# spec.loader.exec_module(module) -# sys.modules[absolute_name] = module -# if path is not None: -# setattr(parent_module, child_name, module) -# return module - - -# if sys.version_info > (3, 4): -# import importlib.util -# -# # Adapted from : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module -# # Useful to debug details... -# def import_module(name, package=None): -# # using find_spec to use our finder -# spec = importlib.util.find_spec(name, package) -# -# # path = None -# # if '.' in absolute_name: -# # parent_name, _, child_name = absolute_name.rpartition('.') -# # # recursive import call -# # parent_module = import_module(parent_name) -# # # getting the path instead of relying on spec (not managed by pytest it seems...) -# # path = [os.path.join(p, child_name) for p in parent_module.__path__ if os.path.exists(os.path.join(p, child_name))] -# -# # getting spec with importlib.util (instead of meta path finder to avoid uncompatible pytest finder) -# #spec = importlib.util.spec_from_file_location(absolute_name, path[0]) -# parent_module = None -# child_name = None -# if spec is None: -# # We didnt find anything, but this is expected on ros packages that haven't been generated yet -# if '.' in name: -# parent_name, _, child_name = name.rpartition('.') -# # recursive import call -# parent_module = import_module(parent_name) -# # we can check if there is a module in there.. -# path = [os.path.join(p, child_name) -# for p in parent_module.__path__._path -# if os.path.exists(os.path.join(p, child_name))] -# # we attempt to get the spec from the first found location -# while path and spec is None: -# spec = importlib.util.spec_from_file_location(name, path[0]) -# path[:] = path[1:] -# else: -# raise ImportError -# -# # checking again in case spec has been modified -# if spec is not None: -# if spec.name not in sys.modules: -# module = importlib.util.module_from_spec(spec) -# spec.loader.exec_module(module) -# sys.modules[name] = module -# if parent_module is not None and child_name is not None: -# setattr(parent_module, child_name, sys.modules[name]) -# return sys.modules[name] -# else: -# raise ImportError -# else: -# def import_module(name, package=None): -# return importlib.import_module(name=name, package=package) - - -# -# Note : we cannot assume anything about import implementation (different python version, different version of pytest) -# => we need to test them all... -# - -# HACK to fix spec from pytest -# @property -# def spec(): -# importlib.util.spec_from_file_location(__file__) - - -def print_importers(): - import sys - import pprint - - print('PATH:'), - pprint.pprint(sys.path) - print() - print('IMPORTERS:') - for name, cache_value in sys.path_importer_cache.items(): - name = name.replace(sys.prefix, '...') - print('%s: %r' % (name, cache_value)) - - -class TestImportRelativeMsgModule(unittest.TestCase): - """Testing custom ROS importer with path already existing and into sys.path""" - - @classmethod - def setUpClass(cls): - # We need to be before FileFinder to be able to find our (non .py[c]) files - # inside, maybe already imported, python packages... - sys.path_hooks.insert(1, rosmsg_finder.ROSFileFinder) - # we do not need to append any path (since we rely on paths already present, and package already found) - - @classmethod - def tearDownClass(cls): - # CAREFUL : Even though we remove the path from sys.path, - # initialized finders will remain in sys.path_importer_cache - sys.path_hooks.remove(rosmsg_finder.ROSFileFinder) - - def test_import_absolute_module(self): - print_importers() - # Verify that files exists and are importable - import pyros_msgs.importer.tests.msg.TestMsg as msg_mod - - self.assertTrue(msg_mod is not None) - self.assertTrue(msg_mod._type == 'pyros_msgs/TestMsg') - - @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_absolute(self): - # Verify that files exists and are importable - importlib.__import__('pyros_msgs.importer.tests.msg.TestMsg',) - msg_mod = sys.modules['pyros_msgs.importer.tests.msg.TestMsg'] - - self.assertTrue(msg_mod is not None) - self.assertTrue(msg_mod._type == 'pyros_msgs/TestMsg') - - @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") - def test_importlib_loadmodule_absolute(self): - # Verify that files exists and are dynamically importable - pkg_list = 'pyros_msgs.importer.tests.msg.TestMsg'.split('.')[:-1] - mod_list = 'pyros_msgs.importer.tests.msg.TestMsg'.split('.')[1:] - pkg = None - for pkg_name, mod_name in zip(pkg_list, mod_list): - pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) - pkg = pkg_loader.load_module(mod_name) - - msg_mod = pkg - - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(msg_mod) - else: - pass - - # TODO : dynamic using specs (python 3.5) - - @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") - def test_importlib_importmodule_absolute(self): - # Verify that files exists and are dynamically importable - msg_mod = importlib.import_module('pyros_msgs.importer.tests.msg.TestMsg') - - self.assertTrue(msg_mod is not None) - - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(msg_mod) - else: - pass - - self.assertTrue(msg_mod is not None) - - -# def test_importlib_msg_module_relative(): -# # Verify that files exists and are importable -# # TODO -# # if hasattr(importlib, 'find_loader'): # recent version of import lib -# # -# # pkg_loader = importlib.find_loader('test_gen_msgs') -# # pkg = pkg_loader.load_module() -# # -# # subpkg_loader = importlib.find_loader('msg', pkg.__path__) -# # subpkg = subpkg_loader.load_module() -# # -# # loader = importlib.find_loader('TestMsg', subpkg.__path__) -# # m = subpkg_loader.load_module() -# # -# # else: # old version -# msg_mod = importlib.import_module('.msg.TestMsg', package=__package__) - - - - -def test_importlib_srv_module(): - pass - # TODO - # # Verify that files exists and are importable - # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') - - -# imp https://pymotw.com/2/imp/index.html -# TODO - -# def test_imp_msg_module(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.msg.TestMsg') -# -# -# def test_imp_msg_pkg(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.msg') -# -# -# def test_imp_srv_module(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.srv.TestSrv') -# -# -# def test_imp_srv_pkg(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.srv') - - -if __name__ == '__main__': - unittest.main() - #import nose - #nose.main() diff --git a/tests/importer/test_rosmsg_import_msgpkg_relative.py b/tests/importer/test_rosmsg_import_msgpkg_relative.py deleted file mode 100644 index 1bd0d47..0000000 --- a/tests/importer/test_rosmsg_import_msgpkg_relative.py +++ /dev/null @@ -1,275 +0,0 @@ -from __future__ import absolute_import, division, print_function -""" -Testing executing rosmsg_import outside of the package, otherwise the package is already imported. -""" - -import os -import sys -import runpy -import logging.config - -logging.config.dictConfig({ - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - } - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, -}) - -# Since test frameworks (like pytest) play with the import machinery, we cannot use it here... -import unittest - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path - -import importlib - -# Importing importer module -from pyros_msgs.importer import rosmsg_finder - -# importlib -# https://pymotw.com/3/importlib/index.html -# https://pymotw.com/2/importlib/index.html - -# if sys.version_info > (3, 4): -# import importlib.util -# # REF : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module -# # Useful to display details when debugging... -# def import_module(name, package=None): -# """An approximate implementation of import.""" -# absolute_name = importlib.util.resolve_name(name, package) -# try: -# return sys.modules[absolute_name] -# except KeyError: -# pass -# -# path = None -# if '.' in absolute_name: -# parent_name, _, child_name = absolute_name.rpartition('.') -# parent_module = import_module(parent_name) -# path = parent_module.spec.submodule_search_locations # this currently breaks (probably because of pytest custom importer ? ) -# for finder in sys.meta_path: -# spec = finder.find_spec(absolute_name, path) -# if spec is not None: -# break -# else: -# raise ImportError('No module named {absolute_name!r}'.format(**locals())) -# module = importlib.util.module_from_spec(spec) -# spec.loader.exec_module(module) -# sys.modules[absolute_name] = module -# if path is not None: -# setattr(parent_module, child_name, module) -# return module - - -# if sys.version_info > (3, 4): -# import importlib.util -# -# # Adapted from : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module -# # Useful to debug details... -# def import_module(name, package=None): -# # using find_spec to use our finder -# spec = importlib.util.find_spec(name, package) -# -# # path = None -# # if '.' in absolute_name: -# # parent_name, _, child_name = absolute_name.rpartition('.') -# # # recursive import call -# # parent_module = import_module(parent_name) -# # # getting the path instead of relying on spec (not managed by pytest it seems...) -# # path = [os.path.join(p, child_name) for p in parent_module.__path__ if os.path.exists(os.path.join(p, child_name))] -# -# # getting spec with importlib.util (instead of meta path finder to avoid uncompatible pytest finder) -# #spec = importlib.util.spec_from_file_location(absolute_name, path[0]) -# parent_module = None -# child_name = None -# if spec is None: -# # We didnt find anything, but this is expected on ros packages that haven't been generated yet -# if '.' in name: -# parent_name, _, child_name = name.rpartition('.') -# # recursive import call -# parent_module = import_module(parent_name) -# # we can check if there is a module in there.. -# path = [os.path.join(p, child_name) -# for p in parent_module.__path__._path -# if os.path.exists(os.path.join(p, child_name))] -# # we attempt to get the spec from the first found location -# while path and spec is None: -# spec = importlib.util.spec_from_file_location(name, path[0]) -# path[:] = path[1:] -# else: -# raise ImportError -# -# # checking again in case spec has been modified -# if spec is not None: -# if spec.name not in sys.modules: -# module = importlib.util.module_from_spec(spec) -# spec.loader.exec_module(module) -# sys.modules[name] = module -# if parent_module is not None and child_name is not None: -# setattr(parent_module, child_name, sys.modules[name]) -# return sys.modules[name] -# else: -# raise ImportError -# else: -# def import_module(name, package=None): -# return importlib.import_module(name=name, package=package) - - -# -# Note : we cannot assume anything about import implementation (different python version, different version of pytest) -# => we need to test them all... -# - -# HACK to fix spec from pytest -# @property -# def spec(): -# importlib.util.spec_from_file_location(__file__) - - -def print_importers(): - import sys - import pprint - - print('PATH:'), - pprint.pprint(sys.path) - print() - print('IMPORTERS:') - for name, cache_value in sys.path_importer_cache.items(): - name = name.replace(sys.prefix, '...') - print('%s: %r' % (name, cache_value)) - - -class TestImportRelativeMsgPkg(unittest.TestCase): - """Testing custom ROS importer with path already existing and into sys.path""" - - @classmethod - def setUpClass(cls): - # We need to be before FileFinder to be able to find our (non .py[c]) files - # inside, maybe already imported, python packages... - sys.path_hooks.insert(1, rosmsg_finder.ROSFileFinder) - # we do not need to append any path (since we rely on paths already present, and package already found) - - @classmethod - def tearDownClass(cls): - # CAREFUL : Even though we remove the path from sys.path, - # initialized finders will remain in sys.path_importer_cache - sys.path_hooks.remove(rosmsg_finder.ROSFileFinder) - - def test_import_absolute_pkg(self): - print_importers() - # Verify that files exists and are importable - import pyros_msgs.importer.tests.msg as msg_pkg - self.assertTrue(msg_pkg is not None) - self.assertTrue(msg_pkg.TestMsg is not None) - self.assertTrue(msg_pkg.TestMsg._type == 'pyros_msgs/TestMsg') - - @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_absolute(self): - # Verify that files exists and are importable - importlib.__import__('pyros_msgs.importer.tests.msg',) - msg_pkg = sys.modules['pyros_msgs.importer.tests.msg'] - self.assertTrue(msg_pkg is not None) - self.assertTrue(msg_pkg.TestMsg is not None) - self.assertTrue(msg_pkg.TestMsg._type == 'pyros_msgs/TestMsg') - - @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") - def test_importlib_loadmodule_absolute(self): - # Verify that files exists and are dynamically importable - pkg_list = 'pyros_msgs.importer.tests.msg'.split('.')[:-1] - mod_list = 'pyros_msgs.importer.tests.msg'.split('.')[1:] - pkg = None - for pkg_name, mod_name in zip(pkg_list, mod_list): - pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) - pkg = pkg_loader.load_module(mod_name) - - msg_mod = pkg - - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(msg_mod) - else: - pass - - # TODO : dynamic using specs (python 3.5) - - @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") - def test_importlib_importmodule_absolute(self): - # Verify that files exists and are dynamically importable - msg_pkg = importlib.import_module('pyros_msgs.importer.tests.msg') - - self.assertTrue(msg_pkg is not None) - - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(msg_pkg) - else: - pass - - self.assertTrue(msg_pkg is not None) - - -# def test_importlib_msg_module_relative(): -# # Verify that files exists and are importable -# # TODO -# # if hasattr(importlib, 'find_loader'): # recent version of import lib -# # -# # pkg_loader = importlib.find_loader('test_gen_msgs') -# # pkg = pkg_loader.load_module() -# # -# # subpkg_loader = importlib.find_loader('msg', pkg.__path__) -# # subpkg = subpkg_loader.load_module() -# # -# # loader = importlib.find_loader('TestMsg', subpkg.__path__) -# # m = subpkg_loader.load_module() -# # -# # else: # old version -# msg_mod = importlib.import_module('.msg.TestMsg', package=__package__) - - - - -def test_importlib_srv_module(): - pass - # TODO - # # Verify that files exists and are importable - # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') - - -# imp https://pymotw.com/2/imp/index.html -# TODO - -# def test_imp_msg_module(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.msg.TestMsg') -# -# -# def test_imp_msg_pkg(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.msg') -# -# -# def test_imp_srv_module(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.srv.TestSrv') -# -# -# def test_imp_srv_pkg(): -# # Verify that files exists and are importable -# msg_mod = imp.import_module('test_gen_msgs.srv') - - -if __name__ == '__main__': - unittest.main() - #import nose - #nose.main() From 141d5338cdc09a1cc0c886229053de6ff1f46524 Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 23 Jun 2017 14:54:24 +0900 Subject: [PATCH 12/28] unified ROSloaders --- pyros_msgs/importer/rosmsg_loader.py | 275 +++++++++++---------------- 1 file changed, 115 insertions(+), 160 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index bcd62ae..bbf0b79 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -55,166 +55,121 @@ def _get_key_name(fullname, db): import importlib.abc import importlib.machinery - class ROSMsgLoader(importlib.machinery.SourceFileLoader): - - def __init__(self, fullname, path): - - self.logger = logging.getLogger(__name__) - - # Doing this in each loader, in case we are running from different processes, - # avoiding to reload from same file (especially useful for boxed tests). - # But deterministic path to avoid regenerating from the same interpreter - self.rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) - if os.path.exists(self.rosimport_path): - shutil.rmtree(self.rosimport_path) - os.makedirs(self.rosimport_path) - - self.rospackage = fullname.partition('.')[0] - # We should reproduce package structure in generated file structure - dirlist = path.split(os.sep) - pkgidx = dirlist[::-1].index(self.rospackage) - indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != 'msg' and not p.endswith('.msg')] - self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) - - # : hack to be able to import a generated class (if requested) - # self.requested_class = None - - if os.path.isdir(path): - if path.endswith('msg') and any([f.endswith('.msg') for f in os.listdir(path)]): # if we get a non empty 'msg' folder - init_path = os.path.join(self.outdir_pkg, 'msg', '__init__.py') - if not os.path.exists(init_path): - # TODO : we need to determine that from the loader - # as a minimum we need to add current package - self.includepath = [self.rospackage + ':'+ path] - - # TODO : dynamic in memory generation (we do not need the file ultimately...) - self.gen_msgs = rosmsg_generator.genmsg_py( - msg_files=[os.path.join(path, f) for f in os.listdir(path)], # every file not ending in '.msg' will be ignored - package=self.rospackage, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - initpy=True # we always create an __init__.py when called from here. - ) - init_path = None - for pyf in self.gen_msgs: - if pyf.endswith('__init__.py'): - init_path = pyf - - if not init_path: - raise ImportError("__init__.py file not found".format(init_path)) - if not os.path.exists(init_path): - raise ImportError("{0} file not found".format(init_path)) - - # relying on usual source file loader since we have generated normal python code - super(ROSMsgLoader, self).__init__(fullname, init_path) - else: # it is a directory potentially containing an 'msg' - # If we are here, it means it wasn't loaded before - # We need to be able to load from source - super(ROSMsgLoader, self).__init__(fullname, path) - - # or to load from installed ros package (python already generated, no point to generate again) - # Note : the path being in sys.path or not is a matter of ROS setup or metafinder. - # TODO - - elif os.path.isfile(path): - # The file should have already been generated (by the loader for a msg package) - # Note we do not want to rely on namespace packages here, since they are not standardized for python2, - # and they can prevent some useful usecases. - - # Hack to be able to "import generated classes" - modname = fullname.rpartition('.')[2] - filepath = os.path.join(self.outdir_pkg, 'msg', '_' + modname + '.py') # the generated module - # relying on usual source file loader since we have previously generated normal python code - super(ROSMsgLoader, self).__init__(fullname, filepath) - - def get_gen_path(self): - """Returning the generated path matching the import""" - return os.path.join(self.outdir_pkg, 'msg') - - def exec_module(self, module): - super(ROSMsgLoader, self).exec_module(module=module) - - - class ROSSrvLoader(importlib.machinery.SourceFileLoader): - - def __init__(self, fullname, path): - - self.logger = logging.getLogger(__name__) - - # Doing this in each loader, in case we are running from different processes, - # avoiding to reload from same file (especially useful for boxed tests). - # But deterministic path to avoid regenerating from the same interpreter - self.rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) - if os.path.exists(self.rosimport_path): - shutil.rmtree(self.rosimport_path) - os.makedirs(self.rosimport_path) - - self.rospackage = fullname.partition('.')[0] - # We should reproduce package structure in generated file structure - dirlist = path.split(os.sep) - pkgidx = dirlist[::-1].index(self.rospackage) - indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != 'srv' and not p.endswith('.srv')] - self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) - - # : hack to be able to import a generated class (if requested) - # self.requested_class = None - - if os.path.isdir(path): - if path.endswith('srv') and any([f.endswith('.srv') for f in os.listdir(path)]): # if we get a non empty 'msg' folder - init_path = os.path.join(self.outdir_pkg, 'srv', '__init__.py') - if not os.path.exists(init_path): - # TODO : we need to determine that from the loader - # as a minimum we need to add current package - self.includepath = [self.rospackage + ':'+ path] - - # TODO : dynamic in memory generation (we do not need the file ultimately...) - self.gen_srvs = rosmsg_generator.gensrv_py( - srv_files=[os.path.join(path, f) for f in os.listdir(path)], # every file not ending in '.msg' will be ignored - package=self.rospackage, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - initpy=True # we always create an __init__.py when called from here. - ) - init_path = None - for pyf in self.gen_srvs: - if pyf.endswith('__init__.py'): - init_path = pyf - - if not init_path: - raise ImportError("__init__.py file not found".format(init_path)) - if not os.path.exists(init_path): - raise ImportError("{0} file not found".format(init_path)) - - # relying on usual source file loader since we have generated normal python code - super(ROSSrvLoader, self).__init__(fullname, init_path) - else: # it is a directory potentially containing an 'msg' - # If we are here, it means it wasn't loaded before - # We need to be able to load from source - super(ROSSrvLoader, self).__init__(fullname, path) - - # or to load from installed ros package (python already generated, no point to generate again) - # Note : the path being in sys.path or not is a matter of ROS setup or metafinder. - # TODO - - elif os.path.isfile(path): - # The file should have already been generated (by the loader for a msg package) - # Note we do not want to rely on namespace packages here, since they are not standardized for python2, - # and they can prevent some useful usecases. - - # Hack to be able to "import generated classes" - modname = fullname.rpartition('.')[2] - filepath = os.path.join(self.outdir_pkg, 'srv', '_' + modname + '.py') # the generated module - # relying on usual source file loader since we have previously generated normal python code - super(ROSSrvLoader, self).__init__(fullname, filepath) - - def get_gen_path(self): - """Returning the generated path matching the import""" - return os.path.join(self.outdir_pkg, 'srv') - - def exec_module(self, module): - super(ROSSrvLoader, self).exec_module(module=module) - - + def RosLoader(rosdef_extension): + """ + Function generating ROS loaders. + This is used to keep .msg and .srv loaders very similar + """ + if rosdef_extension == '.msg': + loader_origin_subdir = 'msg' + loader_file_extension = rosdef_extension + loader_generated_subdir = 'msg' + elif rosdef_extension == '.srv': + loader_origin_subdir = 'srv' + loader_file_extension = rosdef_extension + loader_generated_subdir = 'srv' + else: + raise RuntimeError("RosLoader for a format {0} other than .msg or .srv is not supported".format(rosdef_extension)) + + class ROSDefLoader(importlib.machinery.SourceFileLoader): + def __init__(self, fullname, path): + + self.logger = logging.getLogger(__name__) + + # Doing this in each loader, in case we are running from different processes, + # avoiding to reload from same file (especially useful for boxed tests). + # But deterministic path to avoid regenerating from the same interpreter + self.rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) + if os.path.exists(self.rosimport_path): + shutil.rmtree(self.rosimport_path) + os.makedirs(self.rosimport_path) + + self.rospackage = fullname.partition('.')[0] + # We should reproduce package structure in generated file structure + dirlist = path.split(os.sep) + pkgidx = dirlist[::-1].index(self.rospackage) + indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != loader_origin_subdir and not p.endswith(loader_file_extension)] + self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) + + # : hack to be able to import a generated class (if requested) + # self.requested_class = None + + if os.path.isdir(path): + if path.endswith(loader_origin_subdir) and any([f.endswith(loader_file_extension) for f in os.listdir(path)]): # if we get a non empty 'msg' folder + init_path = os.path.join(self.outdir_pkg, loader_generated_subdir, '__init__.py') + if not os.path.exists(init_path): + # TODO : we need to determine that from the loader + # as a minimum we need to add current package + self.includepath = [self.rospackage + ':' + path] + + # TODO : unify this after reviewing rosmsg_generator API + if loader_file_extension == '.msg': + # TODO : dynamic in memory generation (we do not need the file ultimately...) + self.gen_msgs = rosmsg_generator.genmsg_py( + msg_files=[os.path.join(path, f) for f in os.listdir(path)], # every file not ending in '.msg' will be ignored + package=self.rospackage, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + initpy=True # we always create an __init__.py when called from here. + ) + init_path = None + for pyf in self.gen_msgs: + if pyf.endswith('__init__.py'): + init_path = pyf + elif loader_file_extension == '.srv': + # TODO : dynamic in memory generation (we do not need the file ultimately...) + self.gen_msgs = rosmsg_generator.gensrv_py( + srv_files=[os.path.join(path, f) for f in os.listdir(path)], + # every file not ending in '.msg' will be ignored + package=self.rospackage, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + initpy=True # we always create an __init__.py when called from here. + ) + init_path = None + for pyf in self.gen_msgs: + if pyf.endswith('__init__.py'): + init_path = pyf + else: + raise RuntimeError( + "RosDefLoader for a format {0} other than .msg or .srv is not supported".format( + rosdef_extension)) + + if not init_path: + raise ImportError("__init__.py file not found".format(init_path)) + if not os.path.exists(init_path): + raise ImportError("{0} file not found".format(init_path)) + + # relying on usual source file loader since we have generated normal python code + super(ROSDefLoader, self).__init__(fullname, init_path) + else: # it is a directory potentially containing an 'msg' + # If we are here, it means it wasn't loaded before + # We need to be able to load from source + super(ROSDefLoader, self).__init__(fullname, path) + + # or to load from installed ros package (python already generated, no point to generate again) + # Note : the path being in sys.path or not is a matter of ROS setup or metafinder. + # TODO + + elif os.path.isfile(path): + # The file should have already been generated (by the loader for a msg package) + # Note we do not want to rely on namespace packages here, since they are not standardized for python2, + # and they can prevent some useful usecases. + + # Hack to be able to "import generated classes" + modname = fullname.rpartition('.')[2] + filepath = os.path.join(self.outdir_pkg, loader_generated_subdir, '_' + modname + '.py') # the generated module + # relying on usual source file loader since we have previously generated normal python code + super(ROSDefLoader, self).__init__(fullname, filepath) + + def get_gen_path(self): + """Returning the generated path matching the import""" + return os.path.join(self.outdir_pkg, loader_generated_subdir) + + return ROSDefLoader + + ROSMsgLoader = RosLoader(rosdef_extension='.msg') + ROSSrvLoader = RosLoader(rosdef_extension='.srv') else: From 48acde681020b245abb019421f337beecec3f6f8 Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 23 Jun 2017 19:34:32 +0900 Subject: [PATCH 13/28] WIP starting to backport for python2.7 --- .gitmodules | 3 + pydeps/importlib2-fixed | 1 + pyros_msgs/importer/importlib2 | 1 + pyros_msgs/importer/rosmsg_finder.py | 533 +++++--------------- pyros_msgs/importer/rosmsg_loader.py | 705 +++++++++++++++++++++------ setup.py | 1 + 6 files changed, 682 insertions(+), 562 deletions(-) create mode 160000 pydeps/importlib2-fixed create mode 120000 pyros_msgs/importer/importlib2 diff --git a/.gitmodules b/.gitmodules index fdd19ae..f4d3401 100644 --- a/.gitmodules +++ b/.gitmodules @@ -13,3 +13,6 @@ [submodule "rosdeps/ros_comm_msgs"] path = rosdeps/ros_comm_msgs url = https://github.com/ros/ros_comm_msgs.git +[submodule "pydeps/importlib2-fixed"] + path = pydeps/importlib2-fixed + url = https://github.com/pombredanne/ericsnowcurrently-importlib2.git diff --git a/pydeps/importlib2-fixed b/pydeps/importlib2-fixed new file mode 160000 index 0000000..3279cad --- /dev/null +++ b/pydeps/importlib2-fixed @@ -0,0 +1 @@ +Subproject commit 3279cadc1d03421072f595a8ee8197919e906b1e diff --git a/pyros_msgs/importer/importlib2 b/pyros_msgs/importer/importlib2 new file mode 120000 index 0000000..fb69b6d --- /dev/null +++ b/pyros_msgs/importer/importlib2 @@ -0,0 +1 @@ +../../pydeps/importlib2-fixed/importlib2 \ No newline at end of file diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py index c8177e3..fb490e1 100644 --- a/pyros_msgs/importer/rosmsg_finder.py +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -25,424 +25,131 @@ from .rosmsg_loader import ROSMsgLoader, ROSSrvLoader -if sys.version_info >= (3, 4): - - import importlib.abc - import importlib.machinery - import importlib.util - - - class DirectoryFinder(importlib.machinery.FileFinder): - """Finder to interpret directories as modules, and files as classes""" - - def __init__(self, path, *ros_loader_details): - """ - Finder to get directories containing ROS message and service files. - It need to be inserted in sys.path_hooks before FileFinder, since these are Directories but not containing __init__ as per python hardcoded convention. - - Note: There is a matching issue between msg/ folder and msg/My.msg on one side, and package, module, class concepts on the other. - Since a module is not callable, but we need to call My(data) to build a message class (ROS convention), we match the msg/ folder to a module (and not a package) - And to keep matching ROS conventions, a directory without __init__ or any message/service file, will become a namespace (sub)package. - - :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) - """ - - ros_loaders = [] - for loader, suffixes in ros_loader_details: - ros_loaders.extend((suffix, loader) for suffix in suffixes) - self._ros_loaders = ros_loaders - - # We rely on FileFinder and python loader to deal with our generated code - super(DirectoryFinder, self).__init__( - path, - (importlib.machinery.SourceFileLoader, ['.py']), - (importlib.machinery.SourcelessFileLoader, ['.pyc']), - ) - - def __repr__(self): - return 'DirectoryFinder({!r})'.format(self.path) - - def find_spec(self, fullname, target=None): - """ - Try to find a spec for the specified module. - Returns the matching spec, or None if not found. - :param fullname: the name of the package we are trying to import - :param target: what we plan to do with it - :return: - """ - - tail_module = fullname.rpartition('.')[2] - - spec = None - - # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. - # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package - # tail_module = tail_module[1:] - base_path = os.path.join(self.path, tail_module) - - # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. - if os.path.isdir(base_path): - loader_class = None - rosdir = None - # Figuring out if we should care about this directory at all - for root, dirs, files in os.walk(base_path): - for suffix, loader_cls in self._ros_loaders: - if any(f.endswith(suffix) for f in files): - loader_class = loader_cls - rosdir = root - if loader_class and rosdir and rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module - loader = loader_class(fullname, base_path) - # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location - spec = importlib.util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) - # We DO NOT WANT TO add the generated dir in sys.path to use a python loader - # since the plan is to eventually not have to rely on files at all TODO +if (2, 7) <= sys.version_info < (3, 4): + from .importlib2 import machinery as importlib_machinery + from .importlib2 import util as importlib_util + import pkg_resources # useful to have empty directory imply namespace package (like for py3) + +elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 but importlib2 theoritically could... + import importlib.machinery as importlib_machinery + import importlib.util as importlib_util + +else: + raise ImportError("ros_loader : Unsupported python version") - # Relying on FileFinder if we couldn't find any specific directory structure/content - # It will return a namespace spec if no file can be found - # or will return a proper loader for already generated python files - spec = spec or super(DirectoryFinder, self).find_spec(fullname, target=target) - # we return None if we couldn't find a spec before - return spec - MSG_SUFFIXES = ['.msg'] - SRV_SUFFIXES = ['.srv'] +class DirectoryFinder(importlib_machinery.FileFinder): + """Finder to interpret directories as modules, and files as classes""" - def _get_supported_ros_loaders(): - """Returns a list of file-based module loaders. - Each item is a tuple (loader, suffixes). + def __init__(self, path, *ros_loader_details): """ - msg = ROSMsgLoader, MSG_SUFFIXES - srv = ROSSrvLoader, SRV_SUFFIXES - return [msg, srv] - - - def _install(): - """Install the path-based import components.""" - supported_loaders = _get_supported_ros_loaders() - sys.path_hooks.extend([DirectoryFinder.path_hook(*supported_loaders)]) - # TODO : sys.meta_path.append(DistroFinder) - - - class ROSImportFinder(importlib.abc.PathEntryFinder): - # TODO : we can use this to enable ROS Finder only on relevant paths (ros distro paths + dev workspaces) from sys.path - - def __init__(self, workspace_entry): - """ - Finder to get ROS packages (python code and message files). - It need to be inserted in sys.path_hooks before FileFinder, since these are Files but not python - - Note : it is currently not possible (tested on python 3.4) to build a solid finder only for a set of messages that would be part of a python package, - since sub-packages do not follow the same import logic as modules (finder not called since we are already with a known standard python importer) - Therefore this finder focuses on importing a whole ROS package, and messages have to be in a ROS package. - - :param path_entry: the ROS workspace path (no finder should have been instantiated yet) - """ - super(ROSImportFinder, self).__init__() - - # First we need to skip all the cases that we are not concerned with - - # if path_entry != self.PATH_TRIGGER: - # self.logger.debug('ROSImportFinder does not work for %s' % path_entry) - # raise ImportError() - - # Adding the share path (where we can find all packages and their messages) - share_path = os.path.join(workspace_entry, 'share') - if os.path.exists(share_path): - self.share_path = share_path - - python_libpath = os.path.join(workspace_entry, 'lib', - 'python' + sys.version_info.major + '.' + sys.version_info.minor) - if os.path.exists(python_libpath): - # adding python site directories for the ROS distro (to find python modules as usual with ROS) - if os.path.exists(os.path.join(python_libpath, 'dist-packages')): - site.addsitedir(os.path.join(python_libpath, 'dist-packages')) - if os.path.exists(os.path.join(python_libpath, 'site-packages')): - site.addsitedir(os.path.join(python_libpath, 'site-packages')) - # Note : here we want to keep a pure python logic to not pollute the environment (not like with pyros-setup) - - # self.rospkgs = [] - # for root, dirs, files in os.walk(self.share_path, topdown=False): - # if 'package.xml' in files: # we have found a ros package - # - # - # - # self.rospkgs.append(root) - # found_one = True - # if not found_one: - # raise ImportError("No ROS package found in {0}".format(workspace_entry)) - - else: # this is not a workspace, maybe we are expected to get the package directly from source ? - found_one = False - self.src_rospkgs = [] - for root, dirs, files in os.walk(workspace_entry, topdown=False): - if 'package.xml' in files: # we have found a ros package - self.src_rospkgs.append(root) - found_one = True - if not found_one: - raise ImportError("No ROS package found in {0}".format(workspace_entry)) - - - # # path_entry contains the path where the finder has been instantiated... - # if (not os.path.exists(path_entry) or # non existent path - # not os.path.basename(path_entry) in ['msg', 'srv'] # path not terminated with msg/srv (msg and srv inside python source) - # ): - # raise ImportError # we raise if we cannot find msg or srv folder - - # Then we can do the initialisation - self.logger = logging.getLogger(__name__) - self.logger.debug('Checking ROSImportFinder support for %s' % workspace_entry) - - self.workspace_entry = workspace_entry - - - def find_spec(self, fullname, target = None): - print('ROSImportFinder looking for "%s"' % fullname) - """ - :param fullname: the name of the package we are trying to import - :param target: what we plan to do with it - :return: - """ - # TODO: read PEP 420 :) - last_mile = fullname.split('.')[-1] - - for src_pkg in self.src_rospkgs: - if os.path.basename(src_pkg) == fullname: # we found it - return importlib.machinery.ModuleSpec( - fullname, - None, # TODO loader - origin=src_pkg, - is_package=True - ) - - # If we get here, we need to find the package in a workspace... - - # If we can find it in share/ with messages, we can generate them. - # good or bad idea ? - # Else, we can use the usual python import (since we added them to site) - #return importlib.util.. (fullname). - - - - for root, dirs, files in os.walk(self.workspace_entry, topdown=False): - if last_mile + '.msg' in files: - return importlib.machinery.ModuleSpec( - fullname, - None, # TODO loader - origin=root, - is_package=False - ) - if last_mile + '.srv' in files: - return importlib.machinery.ModuleSpec( - fullname, - None, # TODO loader - origin=root, - is_package=False - ) - - # we couldn't find the module. let someone else find it. - return None + Finder to get directories containing ROS message and service files. + It need to be inserted in sys.path_hooks before FileFinder, since these are Directories but not containing __init__ as per python hardcoded convention. + Note: There is a matching issue between msg/ folder and msg/My.msg on one side, and package, module, class concepts on the other. + Since a module is not callable, but we need to call My(data) to build a message class (ROS convention), we match the msg/ folder to a module (and not a package) + And to keep matching ROS conventions, a directory without __init__ or any message/service file, will become a namespace (sub)package. + + :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) + """ + + ros_loaders = [] + for loader, suffixes in ros_loader_details: + ros_loaders.extend((suffix, loader) for suffix in suffixes) + self._ros_loaders = ros_loaders + + # We rely on FileFinder and python loader to deal with our generated code + super(DirectoryFinder, self).__init__( + path, + (importlib_machinery.SourceFileLoader, ['.py']), + (importlib_machinery.SourcelessFileLoader, ['.pyc']), + ) + + def __repr__(self): + return 'DirectoryFinder({!r})'.format(self.path) + + if (2, 7) <= sys.version_info < (3, 4): + def find_module(self, fullname, path=None): + from .rosmsg_loader import _NamespaceLoader + spec = self.find_spec(fullname=fullname) + loader = spec.loader + if loader is None: + # A backward compatibility hack from importlib2 + if spec.submodule_search_locations is not None: + loader = _NamespaceLoader(spec.name, spec.origin, spec.submodule_search_locations) + return loader + + def find_spec(self, fullname, target=None): + """ + Try to find a spec for the specified module. + Returns the matching spec, or None if not found. + :param fullname: the name of the package we are trying to import + :param target: what we plan to do with it + :return: + """ + + tail_module = fullname.rpartition('.')[2] + + spec = None + + # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. + # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package + # tail_module = tail_module[1:] + base_path = os.path.join(self.path, tail_module) + + # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. + if os.path.isdir(base_path): + loader_class = None + rosdir = None + # Figuring out if we should care about this directory at all + for root, dirs, files in os.walk(base_path): + for suffix, loader_cls in self._ros_loaders: + if any(f.endswith(suffix) for f in files): + loader_class = loader_cls + rosdir = root + if loader_class and rosdir: + if rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module + # Generate something ! + loader = loader_class(fullname, base_path) + # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location + spec = importlib_util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) + # We DO NOT WANT TO add the generated dir in sys.path to use a python loader + # since the plan is to eventually not have to rely on files at all TODO + # elif (2, 7) <= sys.version_info < (3, 4): + # # Found nothing, so this should be a namespace, and has to be explicit in python < 3.2 + # # But we still rely on importlib2 code to backport namespace packages logic + # # Done via loader custom code + # spec = importlib_util.spec_from_file_location(fullname, base_path, + # loader=None, + # #loader=loader_class(fullname, base_path, implicit_ns_pkg=True) + # submodule_search_locations=[base_path]) + # #spec = importlib_machinery.ModuleSpec(name=fullname, origin=base_path, loader=loader_class(fullname, base_path, implicit_ns_pkg=True), is_package=True) + + # Relying on FileFinder if we couldn't find any specific directory structure/content + # It will return a namespace spec if no file can be found (even with python2.7, thanks to importlib2) + # or will return a proper loader for already generated python files + spec = spec or super(DirectoryFinder, self).find_spec(fullname, target=target) + # we return None if we couldn't find a spec before + return spec + +MSG_SUFFIXES = ['.msg'] +SRV_SUFFIXES = ['.srv'] + +def _get_supported_ros_loaders(): + """Returns a list of file-based module loaders. + Each item is a tuple (loader, suffixes). + """ + msg = ROSMsgLoader, MSG_SUFFIXES + srv = ROSSrvLoader, SRV_SUFFIXES + return [msg, srv] + + +def _install(): + """Install the path-based import components.""" + supported_loaders = _get_supported_ros_loaders() + sys.path_hooks.extend([DirectoryFinder.path_hook(*supported_loaders)]) + # TODO : sys.meta_path.append(DistroFinder) -else: - class ROSImportFinder(object): - """Find ROS message/service modules""" - def __init__(self, path_entry=None): - # First we need to skip all the cases that we are not concerned with - - # path_entry contains the path where the finder has been instantiated... - if not os.path.exists(os.path.join(path_entry, 'msg')) and not os.path.exists(os.path.join(path_entry, 'srv')): - raise ImportError # we raise if we cannot find msg or srv folder - - # Then we can do the initialisation - self.logger = logging.getLogger(__name__) - self.logger.debug('Checking ROSImportFinder support for %s' % path_entry) - - self.path_entry = path_entry - - - # This is called for the first unknown (not loaded) name inside path - def find_module(self, name, path=None): - self.logger.debug('ROSImportFinder looking for "%s"' % name) - - # on 2.7 - path = path or self.path_entry - - # implementation inspired from pytest.rewrite - names = name.rsplit(".", 1) - lastname = names[-1] - - - if not os.path.exists(os.path.join(path, lastname)): - raise ImportError - elif os.path.isdir(os.path.join(path, lastname)): - - rosf = [f for f in os.listdir(os.path.join(path, lastname)) if os.path.splitext(f)[-1] in ['.msg', '.srv']] - - if rosf: - return ROSLoader( - msgsrv_files=rosf, - path_entry=os.path.join(path, lastname), - package=name, - outdir_pkg=path, # TODO: if we cannot write into source, use tempfile - #includepath=, - #ns_pkg=, - ) - - # # package case - # for root, dirs, files in os.walk(path, topdown=True): - # - # #rosmsg_generator.genmsgsrv_py(msgsrv_files=[f for f in files if ], package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) - # - # # generated_msg = genmsg_py(msg_files=[f for f in files if f.endswith('.msg')], - # # package=package, - # # outdir_pkg=outdir_pkg, - # # includepath=includepath, - # # initpy=True) # we always create an __init__.py when called from here. - # # generated_srv = gensrv_py(srv_files=[f for f in files if f.endswith('.srv')], - # # package=package, - # # outdir_pkg=outdir_pkg, - # # includepath=includepath, - # # initpy=True) # we always create an __init__.py when called from here. - # - # for name in dirs: - # print(os.path.join(root, name)) - # # rosmsg_generator.genmsgsrv_py(msgsrv_files=msgsrvfiles, package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) - - elif os.path.isfile(os.path.join(path, lastname)): - # module case - return ROSLoader( - msgsrv_files=os.path.join(path, lastname), - path_entry=path, - package=name, - outdir_pkg=path, # TODO: if we cannot write into source, use tempfile - # includepath=, - # ns_pkg=, - ) - - return None - - - - - - # def find_module(self, name, path=None): - # """ - # Return the loader for the specified module. - # """ - # # Ref : https://www.python.org/dev/peps/pep-0302/#specification-part-1-the-importer-protocol - # - # # - # loader = None - # - # # path = path or sys.path - # # for p in path: - # # for f in os.listdir(p): - # # filename, ext = os.path.splitext(f) - # # # our modules generated from messages are always a leaf in import tree so we only care about this case - # # if ext in self.loaders.keys() and filename == name.split('.')[-1]: - # # loader = self.loaders.get(ext) - # # break # we found it. break out. - # # - # # return loader - # - # # implementation inspired from pytest.rewrite - # self.logger.debug("find_module called for: %s" % name) - # names = name.rsplit(".", 1) - # lastname = names[-1] - # pth = None - # if path is not None: - # # Starting with Python 3.3, path is a _NamespacePath(), which - # # causes problems if not converted to list. - # path = list(path) - # if len(path) == 1: - # pth = path[0] - # - # - # if pth is None: - # - # - # - # - # - # try: - # fd, fn, desc = imp.find_module(lastname, path) - # except ImportError: - # return None - # if fd is not None: - # fd.close() - # - # - # - # - # tp = desc[2] - # if tp == imp.PY_COMPILED: - # if hasattr(imp, "source_from_cache"): - # try: - # fn = imp.source_from_cache(fn) - # except ValueError: - # # Python 3 doesn't like orphaned but still-importable - # # .pyc files. - # fn = fn[:-1] - # else: - # fn = fn[:-1] - # elif tp != imp.PY_SOURCE: - # # Don't know what this is. - # return None - # else: - # fn = os.path.join(pth, name.rpartition(".")[2] + ".py") - # - # fn_pypath = py.path.local(fn) - # if not self._should_rewrite(name, fn_pypath, state): - # return None - # - # self._rewritten_names.add(name) - # - # # The requested module looks like a test file, so rewrite it. This is - # # the most magical part of the process: load the source, rewrite the - # # asserts, and load the rewritten source. We also cache the rewritten - # # module code in a special pyc. We must be aware of the possibility of - # # concurrent pytest processes rewriting and loading pycs. To avoid - # # tricky race conditions, we maintain the following invariant: The - # # cached pyc is always a complete, valid pyc. Operations on it must be - # # atomic. POSIX's atomic rename comes in handy. - # write = not sys.dont_write_bytecode - # cache_dir = os.path.join(fn_pypath.dirname, "__pycache__") - # if write: - # try: - # os.mkdir(cache_dir) - # except OSError: - # e = sys.exc_info()[1].errno - # if e == errno.EEXIST: - # # Either the __pycache__ directory already exists (the - # # common case) or it's blocked by a non-dir node. In the - # # latter case, we'll ignore it in _write_pyc. - # pass - # elif e in [errno.ENOENT, errno.ENOTDIR]: - # # One of the path components was not a directory, likely - # # because we're in a zip file. - # write = False - # elif e in [errno.EACCES, errno.EROFS, errno.EPERM]: - # state.trace("read only directory: %r" % fn_pypath.dirname) - # write = False - # else: - # raise - # cache_name = fn_pypath.basename[:-3] + PYC_TAIL - # pyc = os.path.join(cache_dir, cache_name) - # # Notice that even if we're in a read-only directory, I'm going - # # to check for a cached pyc. This may not be optimal... - # co = _read_pyc(fn_pypath, pyc, state.trace) - # if co is None: - # state.trace("rewriting %r" % (fn,)) - # source_stat, co = _rewrite_test(self.config, fn_pypath) - # if co is None: - # # Probably a SyntaxError in the test. - # return None - # if write: - # _make_rewritten_pyc(state, source_stat, pyc, co) - # else: - # state.trace("found cached rewritten pyc for %r" % (fn,)) - # self.modules[name] = co, pyc - # return self # Useless ? @@ -450,7 +157,7 @@ def find_module(self, name, path=None): ros_distro_finder = None - +# TODO : metafinder def activate(rosdistro_path=None, *workspaces): global ros_distro_finder if rosdistro_path is None: # autodetect most recent installed distro diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index bbf0b79..70812e5 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -30,52 +30,553 @@ import logging +def _verbose_message(message, *args, **kwargs): + """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" + verbosity = kwargs.pop('verbosity', 1) + if sys.flags.verbose >= verbosity: + if not message.startswith(('#', 'import ')): + message = '# ' + message + print(message.format(*args), file=sys.stderr) + +# We extend importlib machinery to interpret some specific directoty with specific content as a module +# This needs to match the logic in setup.py +if (2, 7) <= sys.version_info < (3, 4): + from .importlib2 import machinery as importlib_machinery + from .importlib2 import util as importlib_util + + # Module-level locking ######################################################## + + # A dict mapping module names to weakrefs of _ModuleLock instances + _module_locks = {} + # A dict mapping thread ids to _ModuleLock instances + _blocking_on = {} + + # The following two functions are for consumption by Python/import.c. + + # def _get_module_lock(name): + # """Get or create the module lock for a given module name. + # + # Should only be called with the import lock taken.""" + # lock = None + # try: + # lock = _module_locks[name]() + # except KeyError: + # pass + # if lock is None: + # if _thread is None: + # lock = _DummyModuleLock(name) + # else: + # lock = _ModuleLock(name) + # + # def cb(_): + # del _module_locks[name] + # + # _module_locks[name] = _weakref.ref(lock, cb) + # return lock + + + # def _lock_unlock_module(name): + # """Release the global import lock, and acquires then release the + # module lock for a given module name. + # This is used to ensure a module is completely initialized, in the + # event it is being imported by another thread. + # + # Should only be called with the import lock taken.""" + # lock = _get_module_lock(name) + # _imp.release_lock() + # try: + # lock.acquire() + # except _DeadlockError: + # # Concurrent circular import, we'll accept a partially initialized + # # module object. + # pass + # else: + # lock.release() + + # class _ModuleLockManager(object): + # + # def __init__(self, name): + # self._name = name + # self._lock = None + # + # def __enter__(self): + # try: + # self._lock = _get_module_lock(self._name) + # finally: + # _imp.release_lock() + # self._lock.acquire() + # + # def __exit__(self, *args, **kwargs): + # self._lock.release() + + class _SpecMethods(object): + + """Convenience wrapper around spec objects to provide spec-specific + methods.""" + + # The various spec_from_* functions could be made factory methods here. + + def __init__(self, spec): + self.spec = spec + + def module_repr(self): + """Return the repr to use for the module.""" + # We mostly replicate _module_repr() using the spec attributes. + spec = self.spec + name = '?' if spec.name is None else spec.name + if spec.origin is None: + if spec.loader is None: + return ''.format(name) + else: + return ''.format(name, spec.loader) + else: + if spec.has_location: + return ''.format(name, spec.origin) + else: + return ''.format(spec.name, spec.origin) + + def init_module_attrs(self, module, _override=False, _force_name=True): + """Set the module's attributes. + + All missing import-related module attributes will be set. Here + is how the spec attributes map onto the module: + + spec.name -> module.__name__ + spec.loader -> module.__loader__ + spec.parent -> module.__package__ + spec -> module.__spec__ + + Optional: + spec.origin -> module.__file__ (if spec.set_fileattr is true) + spec.cached -> module.__cached__ (if __file__ also set) + spec.submodule_search_locations -> module.__path__ (if set) + + """ + spec = self.spec + + # The passed in module may be not support attribute assignment, + # in which case we simply don't set the attributes. + + # __name__ + if (_override or _force_name or + getattr(module, '__name__', None) is None): + try: + module.__name__ = spec.name + except AttributeError: + pass + + # __loader__ + if _override or getattr(module, '__loader__', None) is None: + loader = spec.loader + if loader is None: + # A backward compatibility hack. + if spec.submodule_search_locations is not None: + loader = _NamespaceLoader.__new__(_NamespaceLoader) + loader._path = spec.submodule_search_locations + try: + module.__loader__ = loader + except AttributeError: + pass + + # __package__ + if _override or getattr(module, '__package__', None) is None: + try: + module.__package__ = spec.parent + except AttributeError: + pass + + # __spec__ + try: + module.__spec__ = spec + except AttributeError: + pass -def _mk_init_name(fullname): - """Return the name of the __init__ module for a given package name.""" - if fullname.endswith('.__init__'): - return fullname - return fullname + '.__init__' - - -def _get_key_name(fullname, db): - """Look in an open shelf for fullname or fullname.__init__, return the name found.""" - if fullname in db: - return fullname - init_name = _mk_init_name(fullname) - if init_name in db: - return init_name - return None - - - + # __path__ + if _override or getattr(module, '__path__', None) is None: + if spec.submodule_search_locations is not None: + try: + module.__path__ = spec.submodule_search_locations + except AttributeError: + pass + + if spec.has_location: + # __file__ + if _override or getattr(module, '__file__', None) is None: + try: + module.__file__ = spec.origin + except AttributeError: + pass + + # __cached__ + if _override or getattr(module, '__cached__', None) is None: + if spec.cached is not None: + try: + module.__cached__ = spec.cached + except AttributeError: + pass + + def create(self): + """Return a new module to be loaded. + + The import-related module attributes are also set with the + appropriate values from the spec. + + """ + spec = self.spec + # Typically loaders will not implement create_module(). + if hasattr(spec.loader, 'create_module'): + # If create_module() returns `None` it means the default + # module creation should be used. + module = spec.loader.create_module(spec) + else: + module = None + if module is None: + # This must be done before open() is ever called as the 'io' + # module implicitly imports 'locale' and would otherwise + # trigger an infinite loop. + module = _new_module(spec.name) + self.init_module_attrs(module) + return module + + def _exec(self, module): + """Do everything necessary to execute the module. + + The namespace of `module` is used as the target of execution. + This method uses the loader's `exec_module()` method. + + """ + self.spec.loader.exec_module(module) + + # Used by importlib.reload() and _load_module_shim(). + def exec_(self, module): + """Execute the spec in an existing module's namespace.""" + name = self.spec.name + _imp.acquire_lock() + with _ModuleLockManager(name): + if sys.modules.get(name) is not module: + msg = 'module {!r} not in sys.modules'.format(name) + raise _ImportError(msg, name=name) + if self.spec.loader is None: + if self.spec.submodule_search_locations is None: + raise _ImportError('missing loader', name=self.spec.name) + # namespace package + self.init_module_attrs(module, _override=True) + return module + self.init_module_attrs(module, _override=True) + if not hasattr(self.spec.loader, 'exec_module'): + # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # have exec_module() implemented, we can add a deprecation + # warning here. + self.spec.loader.load_module(name) + else: + self._exec(module) + return sys.modules[name] + + # def _load_backward_compatible(self): + # # (issue19713) Once BuiltinImporter and ExtensionFileLoader + # # have exec_module() implemented, we can add a deprecation + # # warning here. + # spec = self.spec + # spec.loader.load_module(spec.name) + # # The module must be in sys.modules at this point! + # module = sys.modules[spec.name] + # if getattr(module, '__loader__', None) is None: + # try: + # module.__loader__ = spec.loader + # except AttributeError: + # pass + # if getattr(module, '__package__', None) is None: + # try: + # # Since module.__path__ may not line up with + # # spec.submodule_search_paths, we can't necessarily rely + # # on spec.parent here. + # module.__package__ = module.__name__ + # if not hasattr(module, '__path__'): + # module.__package__ = spec.name.rpartition('.')[0] + # except AttributeError: + # pass + # if getattr(module, '__spec__', None) is None: + # try: + # module.__spec__ = spec + # except AttributeError: + # pass + # return module + + # def _load_unlocked(self): + # # A helper for direct use by the import system. + # if self.spec.loader is not None: + # # not a namespace package + # if not hasattr(self.spec.loader, 'exec_module'): + # return self._load_backward_compatible() + # + # module = self.create() + # with _installed_safely(module): + # if self.spec.loader is None: + # if self.spec.submodule_search_locations is None: + # raise _ImportError('missing loader', name=self.spec.name) + # # A namespace package so do nothing. + # else: + # self._exec(module) + # + # # We don't ensure that the import-related module attributes get + # # set in the sys.modules replacement case. Such modules are on + # # their own. + # return sys.modules[self.spec.name] + + # A method used during testing of _load_unlocked() and by + # _load_module_shim(). + def load(self): + """Return a new module object, loaded by the spec's loader. + + The module is not added to its parent. + + If a module is already in sys.modules, that existing module gets + clobbered. + + """ + _imp.acquire_lock() + with _ModuleLockManager(self.spec.name): + return self._load_unlocked() + + def _new_module(name): + return type(sys)(name) + + # inspired from importlib2 + class _NamespacePath(object): + """Represents a namespace package's path. It uses the module name + to find its parent module, and from there it looks up the parent's + __path__. When this changes, the module's own path is recomputed, + using path_finder. For top-level modules, the parent module's path + is sys.path.""" + + def __init__(self, name, path, path_finder): + self._name = name + self._path = path + self._last_parent_path = tuple(self._get_parent_path()) + self._path_finder = path_finder + + def _find_parent_path_names(self): + """Returns a tuple of (parent-module-name, parent-path-attr-name)""" + parent, dot, me = self._name.rpartition('.') + if dot == '': + # This is a top-level module. sys.path contains the parent path. + return 'sys', 'path' + # Not a top-level module. parent-module.__path__ contains the + # parent path. + return parent, '__path__' + + def _get_parent_path(self): + parent_module_name, path_attr_name = self._find_parent_path_names() + return getattr(sys.modules[parent_module_name], path_attr_name) + + def _recalculate(self): + # If the parent's path has changed, recalculate _path + parent_path = tuple(self._get_parent_path()) # Make a copy + if parent_path != self._last_parent_path: + spec = self._path_finder(self._name, parent_path) + # Note that no changes are made if a loader is returned, but we + # do remember the new parent path + if spec is not None and spec.loader is None: + if spec.submodule_search_locations: + self._path = spec.submodule_search_locations + self._last_parent_path = parent_path # Save the copy + return self._path + + def __iter__(self): + return iter(self._recalculate()) + + def __len__(self): + return len(self._recalculate()) + + def __repr__(self): + return '_NamespacePath({!r})'.format(self._path) + + def __contains__(self, item): + return item in self._recalculate() + + def append(self, item): + self._path.append(item) + + # inspired from importlib2 + class _NamespaceLoader(object): + def __init__(self, name, path, subdirs): + self._path = subdirs + + @classmethod + def module_repr(cls, module): + """Return repr for the module. + + The method is deprecated. The import machinery does the job itself. + + """ + return ''.format(module.__name__) + + def is_package(self, fullname): + return True -if sys.version_info >= (3, 4): + def get_source(self, fullname): + return '' + + def get_code(self, fullname): + return compile('', '', 'exec', dont_inherit=True) + + def exec_module(self, module): + pass + + def _init_module_attrs(self, spec, module, _override=False, _force_name=True): + """Set the module's attributes. + + All missing import-related module attributes will be set. Here + is how the spec attributes map onto the module: + + spec.name -> module.__name__ + spec.loader -> module.__loader__ + spec.parent -> module.__package__ + spec -> module.__spec__ + + Optional: + spec.origin -> module.__file__ (if spec.set_fileattr is true) + spec.cached -> module.__cached__ (if __file__ also set) + spec.submodule_search_locations -> module.__path__ (if set) + + """ + # The passed in module may be not support attribute assignment, + # in which case we simply don't set the attributes. + + # __name__ + if (_override or _force_name or + getattr(module, '__name__', None) is None): + try: + module.__name__ = spec.name + except AttributeError: + pass + + # __loader__ + if _override or getattr(module, '__loader__', None) is None: + loader = spec.loader + if loader is None: + # A backward compatibility hack. + if spec.submodule_search_locations is not None: + loader = _NamespaceLoader.__new__(_NamespaceLoader) + loader._path = spec.submodule_search_locations + try: + module.__loader__ = loader + except AttributeError: + pass + + # __package__ + if _override or getattr(module, '__package__', None) is None: + try: + module.__package__ = spec.parent + except AttributeError: + pass + + # __spec__ + try: + module.__spec__ = spec + except AttributeError: + pass - import importlib.abc - import importlib.machinery + # __path__ + if _override or getattr(module, '__path__', None) is None: + if spec.submodule_search_locations is not None: + try: + module.__path__ = spec.submodule_search_locations + except AttributeError: + pass + + if spec.has_location: + # __file__ + if _override or getattr(module, '__file__', None) is None: + try: + module.__file__ = spec.origin + except AttributeError: + pass + + # __cached__ + if _override or getattr(module, '__cached__', None) is None: + if spec.cached is not None: + try: + module.__cached__ = spec.cached + except AttributeError: + pass + + def _create(self, spec): + """Return a new module to be loaded. + + The import-related module attributes are also set with the + appropriate values from the spec. + + """ + # Typically loaders will not implement create_module(). + if hasattr(spec.loader, 'create_module'): + # If create_module() returns `None` it means the default + # module creation should be used. + module = spec.loader.create_module(spec) + else: + module = None + if module is None: + # This must be done before open() is ever called as the 'io' + # module implicitly imports 'locale' and would otherwise + # trigger an infinite loop. + module = _new_module(spec.name) + self._init_module_attrs(spec, module) + return module + + def load_module(self, fullname): + """Load a namespace module. + + This method is deprecated. Use exec_module() instead. + + """ + # The import system never calls this method. + _verbose_message('namespace module loaded with path {!r}', self._path) + + spec = importlib_util.spec_from_loader(fullname, self) + spec.submodule_search_locations = self._path + if fullname in sys.modules: + module = sys.modules[fullname] + self._init_module_attrs(spec, module, _override=True) + return sys.modules[fullname] + else: + module = self._create(spec) + return module + + +elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 but importlib2 theoretically could... + import importlib.machinery as importlib_machinery - def RosLoader(rosdef_extension): - """ - Function generating ROS loaders. - This is used to keep .msg and .srv loaders very similar - """ - if rosdef_extension == '.msg': - loader_origin_subdir = 'msg' - loader_file_extension = rosdef_extension - loader_generated_subdir = 'msg' - elif rosdef_extension == '.srv': - loader_origin_subdir = 'srv' - loader_file_extension = rosdef_extension - loader_generated_subdir = 'srv' - else: - raise RuntimeError("RosLoader for a format {0} other than .msg or .srv is not supported".format(rosdef_extension)) +else: + raise ImportError("ros_loader : Unsupported python version") + + +def RosLoader(rosdef_extension): + """ + Function generating ROS loaders. + This is used to keep .msg and .srv loaders very similar + """ + if rosdef_extension == '.msg': + loader_origin_subdir = 'msg' + loader_file_extension = rosdef_extension + loader_generated_subdir = 'msg' + elif rosdef_extension == '.srv': + loader_origin_subdir = 'srv' + loader_file_extension = rosdef_extension + loader_generated_subdir = 'srv' + else: + raise RuntimeError("RosLoader for a format {0} other than .msg or .srv is not supported".format(rosdef_extension)) + + class ROSDefLoader(importlib_machinery.SourceFileLoader): + def __init__(self, fullname, path, implicit_ns_pkg=False): - class ROSDefLoader(importlib.machinery.SourceFileLoader): - def __init__(self, fullname, path): + self.logger = logging.getLogger(__name__) - self.logger = logging.getLogger(__name__) + self.implicit_ns_package = implicit_ns_pkg + if (2, 7) <= sys.version_info < (3, 4) and implicit_ns_pkg: # the specific python2 usecase where we want a directory to be considered as a namespace package + # we need to use the usual FileFinder logic + super(ROSDefLoader, self).__init__(fullname, path) + else: # Doing this in each loader, in case we are running from different processes, # avoiding to reload from same file (especially useful for boxed tests). # But deterministic path to avoid regenerating from the same interpreter @@ -162,117 +663,23 @@ def __init__(self, fullname, path): # relying on usual source file loader since we have previously generated normal python code super(ROSDefLoader, self).__init__(fullname, filepath) - def get_gen_path(self): - """Returning the generated path matching the import""" - return os.path.join(self.outdir_pkg, loader_generated_subdir) + def get_gen_path(self): + """Returning the generated path matching the import""" + return os.path.join(self.outdir_pkg, loader_generated_subdir) - return ROSDefLoader - - ROSMsgLoader = RosLoader(rosdef_extension='.msg') - ROSSrvLoader = RosLoader(rosdef_extension='.srv') - -else: + def __repr__(self): + return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - class ROSLoader(object): - - def __init__(self, path_entry, msgsrv_files, package, outdir_pkg, includepath=None, ns_pkg=None): - - self.logger = logging.getLogger(__name__) - self.path_entry = path_entry - - self.msgsrv_files = msgsrv_files - self.package = package - self.outdir_pkg = outdir_pkg - - # TODO : we need to determine that from the loader - self.includepath = includepath - self.ns_pkg = ns_pkg - - def _generate(self, fullname): - - gen = rosmsg_generator.genmsgsrv_py( - msgsrv_files=self.msgsrv_files, - package=self.package, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - ns_pkg=self.ns_pkg - ) - return gen - - def get_source(self, fullname): - self.logger.debug('loading source for "%s" from msg' % fullname) - try: - - self._generate() - - # with shelve_context(self.path_entry) as db: - # key_name = _get_key_name(fullname, db) - # if key_name: - # return db[key_name] - # raise ImportError('could not find source for %s' % fullname) - - pass + def exec_module(self, module): + # Custom implementation to declare an implicit namespace package + if (2, 7) <= sys.version_info < (3, 4) and self.implicit_ns_package: + module.__path__ = [self.path] # mandatory to trick importlib2 to think this is a package + import pkg_resources + pkg_resources.declare_namespace(module.__name__) + else: + super(ROSDefLoader, self).exec_module(module) + return ROSDefLoader - except Exception as e: - self.logger.debug('could not load source:', e) - raise ImportError(str(e)) - - # defining this to benefit from backward compat import mechanism in python 3.X - def get_filename(self, name): - os.sep.join(name.split(".")) + '.' + self.ext - - # def _get_filename(self, fullname): - # # Make up a fake filename that starts with the path entry - # # so pkgutil.get_data() works correctly. - # return os.path.join(self.path_entry, fullname) - # - # # defining this to benefit from backward compat import mechanism in python 3.X - # def is_package(self, name): - # names = name.split(".") - # parent_idx = len(names) -1 - # # trying to find a parent already loaded - # while 0<= parent_idx < len(names) : - # if names[parent_idx] in sys.modules: # we found a parent, we can get his path and go back - # pass - # else: # parent not found, need to check its parent - # parent_idx-=1 - # - # - # else: # No loaded parent found, lets attempt to import it directly (searching in all sys.paths) - # - # pass - # - # - # return None # TODO : implement check - # - # def load_module(self, fullname): - # - # if fullname in sys.modules: - # self.logger.debug('reusing existing module from previous import of "%s"' % fullname) - # mod = sys.modules[fullname] - # else: - # self.logger.debug('creating a new module object for "%s"' % fullname) - # mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) - # - # # Set a few properties required by PEP 302 - # mod.__file__ = self._get_filename(fullname) - # mod.__name__ = fullname - # mod.__path__ = self.path_entry - # mod.__loader__ = self - # mod.__package__ = '.'.join(fullname.split('.')[:-1]) - # - # if self.is_package(fullname): - # self.logger.debug('adding path for package') - # # Set __path__ for packages - # # so we can find the sub-modules. - # mod.__path__ = [self.path_entry] - # else: - # self.logger.debug('imported as regular module') - # - # source = self.get_source(fullname) - # - # self.logger.debug('execing source...') - # exec(source, mod.__dict__) - # self.logger.debug('done') - # return mod +ROSMsgLoader = RosLoader(rosdef_extension='.msg') +ROSSrvLoader = RosLoader(rosdef_extension='.srv') diff --git a/setup.py b/setup.py index ad61a1d..e64d9b3 100644 --- a/setup.py +++ b/setup.py @@ -248,6 +248,7 @@ def run(self): # this is needed as install dependency since we embed tests in the package. # 'pyros_setup>=0.2.1', # needed to grab ros environment even if distro setup.sh not sourced # 'pyros_utils', # this must be satisfied by the ROS package system... + # 'importlib2>=3.4;python_version<"3.4"', # NOT working we use a patched version of it, through a symlink (to make linux deb release possible) 'pyyaml>=3.10', # genpy relies on this... 'pytest>=2.8.0', # as per hypothesis requirement (careful with 2.5.1 on trusty) 'pytest-xdist', # for --boxed (careful with the version it will be moved out of xdist) From 392bfc5befe7e5aaf34b5bf129d510d3c74f9e70 Mon Sep 17 00:00:00 2001 From: alexv Date: Mon, 26 Jun 2017 19:20:03 +0900 Subject: [PATCH 14/28] import with implicit namespace package working on 2.7 --- pyros_msgs/importer/rosmsg_finder.py | 560 ++++++++++-- pyros_msgs/importer/rosmsg_loader.py | 809 +++++++----------- .../importer/tests/nspkg/subpkg/__init__.py | 7 + .../importer/tests/nspkg/subpkg/submodule.py | 4 + .../importer/tests/test_rosmsg_import.py | 57 ++ 5 files changed, 845 insertions(+), 592 deletions(-) create mode 100644 pyros_msgs/importer/tests/nspkg/subpkg/__init__.py create mode 100644 pyros_msgs/importer/tests/nspkg/subpkg/submodule.py diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py index fb490e1..84fa8ea 100644 --- a/pyros_msgs/importer/rosmsg_finder.py +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -25,112 +25,496 @@ from .rosmsg_loader import ROSMsgLoader, ROSSrvLoader -if (2, 7) <= sys.version_info < (3, 4): - from .importlib2 import machinery as importlib_machinery - from .importlib2 import util as importlib_util +if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? + # from .importlib2 import machinery as importlib_machinery + # from .importlib2 import util as importlib_util import pkg_resources # useful to have empty directory imply namespace package (like for py3) -elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 but importlib2 theoritically could... - import importlib.machinery as importlib_machinery - import importlib.util as importlib_util + from .rosmsg_loader import _ImportError, _verbose_message, FileLoader2 -else: - raise ImportError("ros_loader : Unsupported python version") + import imp + def spec_from_file_location(name, location=None, loader=None, + submodule_search_locations=None): + """Return a module spec based on a file location. -class DirectoryFinder(importlib_machinery.FileFinder): - """Finder to interpret directories as modules, and files as classes""" + To indicate that the module is a package, set + submodule_search_locations to a list of directory paths. An + empty list is sufficient, though its not otherwise useful to the + import system. + + The loader must take a spec as its only __init__() arg. - def __init__(self, path, *ros_loader_details): """ - Finder to get directories containing ROS message and service files. - It need to be inserted in sys.path_hooks before FileFinder, since these are Directories but not containing __init__ as per python hardcoded convention. - Note: There is a matching issue between msg/ folder and msg/My.msg on one side, and package, module, class concepts on the other. - Since a module is not callable, but we need to call My(data) to build a message class (ROS convention), we match the msg/ folder to a module (and not a package) - And to keep matching ROS conventions, a directory without __init__ or any message/service file, will become a namespace (sub)package. + if location is None: + # The caller may simply want a partially populated location- + # oriented spec. So we set the location to a bogus value and + # fill in as much as we can. + location = '' + if hasattr(loader, 'get_filename'): + # ExecutionLoader + try: + location = loader.get_filename(name) + except ImportError: + pass + + # If the location is on the filesystem, but doesn't actually exist, + # we could return None here, indicating that the location is not + # valid. However, we don't have a good way of testing since an + # indirect location (e.g. a zip file or URL) will look like a + # non-existent file relative to the filesystem. + + spec = ModuleSpec(name, loader, origin=location) + spec._set_fileattr = True + + # Pick a loader if one wasn't provided. + if loader is None: + for loader_class, suffixes in _get_supported_file_loaders(): + if location.endswith(tuple(suffixes)): + loader = loader_class(name, location) + spec.loader = loader + break + else: + return None + + # Set submodule_search_paths appropriately. + if submodule_search_locations is None: + # Check the loader. + if hasattr(loader, 'is_package'): + try: + is_package = loader.is_package(name) + except ImportError: + pass + else: + if is_package: + spec.submodule_search_locations = [] + else: + spec.submodule_search_locations = submodule_search_locations + if spec.submodule_search_locations == []: + if location: + dirname = os.path.split(location)[0] + spec.submodule_search_locations.append(dirname) + + return spec + + class FileFinder2(object): + def __init__(self, path, *loader_details): + """Initialize with the path to search on and a variable number of + 2-tuples containing the loader and the file suffixes the loader + recognizes.""" + loaders = [] + for loader, suffixes in loader_details: + loaders.extend((suffix, loader) for suffix in suffixes) + self._loaders = loaders + # Base (directory) path + self.path = path or '.' + # Note : we are not playing with cache here (too complex to get right and not worth it for obsolete python) + + def _get_spec(self, loader_class, fullname, path, smsl, target): + loader = loader_class(fullname, path) + return spec_from_file_location(fullname, path, loader=loader, submodule_search_locations=smsl) + + def find_module(self, fullname, path=None): + """Try to find a loader for the specified module, or the namespace + package portions. Returns loader.""" + path = path or self.path + tail_module = fullname.rpartition('.')[2] + + base_path = os.path.join(path, tail_module) + for suffix, loader_class in self._loaders: + full_path = None # adjusting path for package or file + if os.path.isdir(base_path) and os.path.isfile(os.path.join(base_path, '__init__' + suffix)): + return loader_class(fullname, base_path) # __init__.py path will be computed by the loader when needed + elif os.path.isfile(base_path + suffix): + return loader_class(fullname, base_path + suffix) + else: + if os.path.isdir(base_path): + # If a namespace package, return the path if we don't + # find a module in the next section. + _verbose_message('possible namespace for {}'.format(base_path)) + return FileLoader2(fullname, base_path) + + # # Check for a file w/ a proper suffix exists. + # for suffix, loader_class in self._loaders: + # full_path = os.path.join(self.path, tail_module + suffix) + # _verbose_message('trying {}'.format(full_path), verbosity=2) + # if cache_module + suffix in cache: + # if os.path.isfile(full_path): + # return loader_class(fullname, full_path) + # #return self._get_spec(loader_class, fullname, full_path, None, target) + + # if is_namespace: + # _verbose_message('possible namespace for {}'.format(base_path)) + # return loader_class(fullname, None) + # # spec = ModuleSpec(fullname, None) + # # spec.submodule_search_locations = [base_path] + # # return spec + return None + + @classmethod + def path_hook(cls, *loader_details): + """A class method which returns a closure to use on sys.path_hook + which will return an instance using the specified loaders and the path + called on the closure. + + If the path called on the closure is not a directory, ImportError is + raised. + + """ + def path_hook_for_FileFinder(path): + """Path hook for importlib.machinery.FileFinder.""" + if not os.path.isdir(path): + raise _ImportError('only directories are supported', path=path) + return cls(path, *loader_details) + + return path_hook_for_FileFinder + + def __repr__(self): + return 'FileFinder2({!r})'.format(self.path) + + + class FileFinder(object): + + """File-based finder. + + Interactions with the file system are cached for performance, being + refreshed when the directory the finder is handling has been modified. + + """ - :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) + def __init__(self, path, *loader_details): + """Initialize with the path to search on and a variable number of + 2-tuples containing the loader and the file suffixes the loader + recognizes.""" + loaders = [] + for loader, suffixes in loader_details: + loaders.extend((suffix, loader) for suffix in suffixes) + self._loaders = loaders + # Base (directory) path + self.path = path or '.' + self._path_mtime = -1 + self._path_cache = set() + self._relaxed_path_cache = set() + + def invalidate_caches(self): + """Invalidate the directory mtime.""" + self._path_mtime = -1 + + #find_module = _find_module_shim + + def find_loader(self, fullname): + """Try to find a loader for the specified module, or the namespace + package portions. Returns (loader, list-of-portions). + + This method is deprecated. Use find_spec() instead. + + """ + spec = self.find_spec(fullname) + if spec is None: + return None, [] + return spec.loader, spec.submodule_search_locations or [] + + + + def find_spec(self, fullname, target=None): + """Try to find a loader for the specified module, or the namespace + package portions. Returns (loader, list-of-portions).""" + is_namespace = False + tail_module = fullname.rpartition('.')[2] + try: + mtime = _path_stat(self.path or _os.getcwd()).st_mtime + except OSError: + mtime = -1 + if mtime != self._path_mtime: + self._fill_cache() + self._path_mtime = mtime + # tail_module keeps the original casing, for __file__ and friends + if _relax_case(): + cache = self._relaxed_path_cache + cache_module = tail_module.lower() + else: + cache = self._path_cache + cache_module = tail_module + # Check if the module is the name of a directory (and thus a package). + if cache_module in cache: + base_path = _path_join(self.path, tail_module) + for suffix, loader_class in self._loaders: + init_filename = '__init__' + suffix + full_path = _path_join(base_path, init_filename) + if _path_isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, [base_path], target) + else: + # If a namespace package, return the path if we don't + # find a module in the next section. + is_namespace = _path_isdir(base_path) + # Check for a file w/ a proper suffix exists. + for suffix, loader_class in self._loaders: + full_path = _path_join(self.path, tail_module + suffix) + _verbose_message('trying {}'.format(full_path), verbosity=2) + if cache_module + suffix in cache: + if _path_isfile(full_path): + return self._get_spec(loader_class, fullname, full_path, None, target) + if is_namespace: + _verbose_message('possible namespace for {}'.format(base_path)) + spec = ModuleSpec(fullname, None) + spec.submodule_search_locations = [base_path] + return spec + return None + + + def _get_supported_ns_loaders(): + """Returns a list of file-based module loaders. + Each item is a tuple (loader, suffixes). """ + loader = FileLoader2, [suffix for suffix, mode, type in imp.get_suffixes()] + return [loader] + + + class DirectoryFinder(FileFinder2): + """Finder to interpret directories as modules, and files as classes""" + + def __init__(self, path, *ros_loader_details): + """ + Finder to get directories containing ROS message and service files. + It need to be inserted in sys.path_hooks before FileFinder, since these are Directories but not containing __init__ as per python hardcoded convention. + + Note: There is a matching issue between msg/ folder and msg/My.msg on one side, and package, module, class concepts on the other. + Since a module is not callable, but we need to call My(data) to build a message class (ROS convention), we match the msg/ folder to a module (and not a package) + And to keep matching ROS conventions, a directory without __init__ or any message/service file, will become a namespace (sub)package. - ros_loaders = [] - for loader, suffixes in ros_loader_details: - ros_loaders.extend((suffix, loader) for suffix in suffixes) - self._ros_loaders = ros_loaders + :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) + """ - # We rely on FileFinder and python loader to deal with our generated code - super(DirectoryFinder, self).__init__( - path, - (importlib_machinery.SourceFileLoader, ['.py']), - (importlib_machinery.SourcelessFileLoader, ['.pyc']), - ) + ros_loaders = [] + for loader, suffixes in ros_loader_details: + ros_loaders.extend((suffix, loader) for suffix in suffixes) + self._ros_loaders = ros_loaders - def __repr__(self): - return 'DirectoryFinder({!r})'.format(self.path) + # # Form importlib FileFinder + # loaders = [] + # for loader, suffixes in [(importlib_machinery.SourceFileLoader, ['.py']), (importlib_machinery.SourcelessFileLoader, ['.pyc'])]: + # loaders.extend((suffix, loader) for suffix in suffixes) + # self._loaders = loaders + # Base (directory) path + self.path = path or '.' + self._path_mtime = -1 + self._path_cache = set() + self._relaxed_path_cache = set() + + def __repr__(self): + return 'DirectoryFinder({!r})'.format(self.path) - if (2, 7) <= sys.version_info < (3, 4): def find_module(self, fullname, path=None): - from .rosmsg_loader import _NamespaceLoader - spec = self.find_spec(fullname=fullname) - loader = spec.loader - if loader is None: - # A backward compatibility hack from importlib2 - if spec.submodule_search_locations is not None: - loader = _NamespaceLoader(spec.name, spec.origin, spec.submodule_search_locations) + path = path or self.path + + tail_module = fullname.rpartition('.')[2] + + loader = None + + # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. + # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package + # tail_module = tail_module[1:] + base_path = os.path.join(self.path, tail_module) + + # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. + if os.path.isdir(base_path): + loader_class = None + rosdir = None + # Figuring out if we should care about this directory at all + for root, dirs, files in os.walk(base_path): + for suffix, loader_cls in self._ros_loaders: + if any(f.endswith(suffix) for f in files): + loader_class = loader_cls + rosdir = root + if loader_class and rosdir: + if rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module + # Generate something ! + # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location + loader = loader_class(fullname, base_path) # loader.get_gen_path()]) + # We DO NOT WANT TO add the generated dir in sys.path to use a python loader + # since the plan is to eventually not have to rely on files at all TODO + else: + # directory not matching our path, so this should be a namespace, and has to be explicit in python < 3.2 + + # But we still rely on importlib2 code to backport namespace packages logic + # Done via loader custom code + loader = loader_class(fullname, base_path, implicit_ns_pkg=True) + + # we return None if we couldn't build a loader before return loader - def find_spec(self, fullname, target=None): - """ - Try to find a spec for the specified module. - Returns the matching spec, or None if not found. - :param fullname: the name of the package we are trying to import - :param target: what we plan to do with it - :return: - """ + # inspired from importlib2 + @classmethod + def path_hook(cls, *loader_details): + """A class method which returns a closure to use on sys.path_hook + which will return an instance using the specified loaders and the path + called on the closure. + + If the path called on the closure is not a directory, ImportError is + raised. + + """ + + def path_hook_for_FileFinder(path): + """Path hook for rosimport.DirectoryFinder.""" + if not os.path.isdir(path): + raise _ImportError('only directories are supported', path=path) + return cls(path, *loader_details) + + return path_hook_for_FileFinder + + def __repr__(self): + return 'FileFinder({!r})'.format(self.path) + + # def find_spec(self, fullname, target=None): + # """ + # Try to find a spec for the specified module. + # Returns the matching spec, or None if not found. + # :param fullname: the name of the package we are trying to import + # :param target: what we plan to do with it + # :return: + # """ + # + # tail_module = fullname.rpartition('.')[2] + # + # spec = None + # + # # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. + # # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package + # # tail_module = tail_module[1:] + # base_path = os.path.join(self.path, tail_module) + # + # # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. + # if os.path.isdir(base_path): + # loader_class = None + # rosdir = None + # # Figuring out if we should care about this directory at all + # for root, dirs, files in os.walk(base_path): + # for suffix, loader_cls in self._ros_loaders: + # if any(f.endswith(suffix) for f in files): + # loader_class = loader_cls + # rosdir = root + # if loader_class and rosdir: + # if rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module + # # Generate something ! + # loader = loader_class(fullname, base_path) + # # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location + # spec = importlib_util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) + # # We DO NOT WANT TO add the generated dir in sys.path to use a python loader + # # since the plan is to eventually not have to rely on files at all TODO + # # elif (2, 7) <= sys.version_info < (3, 4): + # # # Found nothing, so this should be a namespace, and has to be explicit in python < 3.2 + # # # But we still rely on importlib2 code to backport namespace packages logic + # # # Done via loader custom code + # # spec = importlib_util.spec_from_file_location(fullname, base_path, + # # loader=None, + # # #loader=loader_class(fullname, base_path, implicit_ns_pkg=True) + # # submodule_search_locations=[base_path]) + # # #spec = importlib_machinery.ModuleSpec(name=fullname, origin=base_path, loader=loader_class(fullname, base_path, implicit_ns_pkg=True), is_package=True) + # + # # Relying on FileFinder if we couldn't find any specific directory structure/content + # # It will return a namespace spec if no file can be found (even with python2.7, thanks to importlib2) + # # or will return a proper loader for already generated python files + # spec = spec or super(DirectoryFinder, self).find_spec(fullname, target=target) + # # we return None if we couldn't find a spec before + # return spec + + + +elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 but importlib2 theoritically could... + import importlib.machinery as importlib_machinery + import importlib.util as importlib_util + + + + class DirectoryFinder(importlib_machinery.FileFinder): + """Finder to interpret directories as modules, and files as classes""" + + def __init__(self, path, *ros_loader_details): + """ + Finder to get directories containing ROS message and service files. + It need to be inserted in sys.path_hooks before FileFinder, since these are Directories but not containing __init__ as per python hardcoded convention. + + Note: There is a matching issue between msg/ folder and msg/My.msg on one side, and package, module, class concepts on the other. + Since a module is not callable, but we need to call My(data) to build a message class (ROS convention), we match the msg/ folder to a module (and not a package) + And to keep matching ROS conventions, a directory without __init__ or any message/service file, will become a namespace (sub)package. + + :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) + """ + + ros_loaders = [] + for loader, suffixes in ros_loader_details: + ros_loaders.extend((suffix, loader) for suffix in suffixes) + self._ros_loaders = ros_loaders + + # We rely on FileFinder and python loader to deal with our generated code + super(DirectoryFinder, self).__init__( + path, + (importlib_machinery.SourceFileLoader, ['.py']), + (importlib_machinery.SourcelessFileLoader, ['.pyc']), + ) + + def __repr__(self): + return 'DirectoryFinder({!r})'.format(self.path) + + def find_spec(self, fullname, target=None): + """ + Try to find a spec for the specified module. + Returns the matching spec, or None if not found. + :param fullname: the name of the package we are trying to import + :param target: what we plan to do with it + :return: + """ + + tail_module = fullname.rpartition('.')[2] + + spec = None + + # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. + # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package + # tail_module = tail_module[1:] + base_path = os.path.join(self.path, tail_module) + + # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. + if os.path.isdir(base_path): + loader_class = None + rosdir = None + # Figuring out if we should care about this directory at all + for root, dirs, files in os.walk(base_path): + for suffix, loader_cls in self._ros_loaders: + if any(f.endswith(suffix) for f in files): + loader_class = loader_cls + rosdir = root + if loader_class and rosdir: + if rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module + # Generate something ! + loader = loader_class(fullname, base_path) + # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location + spec = importlib_util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) + # We DO NOT WANT TO add the generated dir in sys.path to use a python loader + # since the plan is to eventually not have to rely on files at all TODO + # elif (2, 7) <= sys.version_info < (3, 4): + # # Found nothing, so this should be a namespace, and has to be explicit in python < 3.2 + # # But we still rely on importlib2 code to backport namespace packages logic + # # Done via loader custom code + # spec = importlib_util.spec_from_file_location(fullname, base_path, + # loader=None, + # #loader=loader_class(fullname, base_path, implicit_ns_pkg=True) + # submodule_search_locations=[base_path]) + # #spec = importlib_machinery.ModuleSpec(name=fullname, origin=base_path, loader=loader_class(fullname, base_path, implicit_ns_pkg=True), is_package=True) + + # Relying on FileFinder if we couldn't find any specific directory structure/content + # It will return a namespace spec if no file can be found (even with python2.7, thanks to importlib2) + # or will return a proper loader for already generated python files + spec = spec or super(DirectoryFinder, self).find_spec(fullname, target=target) + # we return None if we couldn't find a spec before + return spec + + +else: + raise ImportError("ros_loader : Unsupported python version") - tail_module = fullname.rpartition('.')[2] - - spec = None - - # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. - # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package - # tail_module = tail_module[1:] - base_path = os.path.join(self.path, tail_module) - - # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. - if os.path.isdir(base_path): - loader_class = None - rosdir = None - # Figuring out if we should care about this directory at all - for root, dirs, files in os.walk(base_path): - for suffix, loader_cls in self._ros_loaders: - if any(f.endswith(suffix) for f in files): - loader_class = loader_cls - rosdir = root - if loader_class and rosdir: - if rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module - # Generate something ! - loader = loader_class(fullname, base_path) - # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location - spec = importlib_util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) - # We DO NOT WANT TO add the generated dir in sys.path to use a python loader - # since the plan is to eventually not have to rely on files at all TODO - # elif (2, 7) <= sys.version_info < (3, 4): - # # Found nothing, so this should be a namespace, and has to be explicit in python < 3.2 - # # But we still rely on importlib2 code to backport namespace packages logic - # # Done via loader custom code - # spec = importlib_util.spec_from_file_location(fullname, base_path, - # loader=None, - # #loader=loader_class(fullname, base_path, implicit_ns_pkg=True) - # submodule_search_locations=[base_path]) - # #spec = importlib_machinery.ModuleSpec(name=fullname, origin=base_path, loader=loader_class(fullname, base_path, implicit_ns_pkg=True), is_package=True) - - # Relying on FileFinder if we couldn't find any specific directory structure/content - # It will return a namespace spec if no file can be found (even with python2.7, thanks to importlib2) - # or will return a proper loader for already generated python files - spec = spec or super(DirectoryFinder, self).find_spec(fullname, target=target) - # we return None if we couldn't find a spec before - return spec MSG_SUFFIXES = ['.msg'] SRV_SUFFIXES = ['.srv'] diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index 70812e5..2c6fd71 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -7,6 +7,7 @@ import shutil + from pyros_msgs.importer import rosmsg_generator """ @@ -38,545 +39,342 @@ def _verbose_message(message, *args, **kwargs): message = '# ' + message print(message.format(*args), file=sys.stderr) -# We extend importlib machinery to interpret some specific directoty with specific content as a module -# This needs to match the logic in setup.py -if (2, 7) <= sys.version_info < (3, 4): - from .importlib2 import machinery as importlib_machinery - from .importlib2 import util as importlib_util - - # Module-level locking ######################################################## - - # A dict mapping module names to weakrefs of _ModuleLock instances - _module_locks = {} - # A dict mapping thread ids to _ModuleLock instances - _blocking_on = {} - - # The following two functions are for consumption by Python/import.c. - - # def _get_module_lock(name): - # """Get or create the module lock for a given module name. - # - # Should only be called with the import lock taken.""" - # lock = None - # try: - # lock = _module_locks[name]() - # except KeyError: - # pass - # if lock is None: - # if _thread is None: - # lock = _DummyModuleLock(name) - # else: - # lock = _ModuleLock(name) - # - # def cb(_): - # del _module_locks[name] - # - # _module_locks[name] = _weakref.ref(lock, cb) - # return lock - - - # def _lock_unlock_module(name): - # """Release the global import lock, and acquires then release the - # module lock for a given module name. - # This is used to ensure a module is completely initialized, in the - # event it is being imported by another thread. - # - # Should only be called with the import lock taken.""" - # lock = _get_module_lock(name) - # _imp.release_lock() - # try: - # lock.acquire() - # except _DeadlockError: - # # Concurrent circular import, we'll accept a partially initialized - # # module object. - # pass - # else: - # lock.release() - - # class _ModuleLockManager(object): - # - # def __init__(self, name): - # self._name = name - # self._lock = None - # - # def __enter__(self): - # try: - # self._lock = _get_module_lock(self._name) - # finally: - # _imp.release_lock() - # self._lock.acquire() - # - # def __exit__(self, *args, **kwargs): - # self._lock.release() - - class _SpecMethods(object): - - """Convenience wrapper around spec objects to provide spec-specific - methods.""" - - # The various spec_from_* functions could be made factory methods here. - - def __init__(self, spec): - self.spec = spec - - def module_repr(self): - """Return the repr to use for the module.""" - # We mostly replicate _module_repr() using the spec attributes. - spec = self.spec - name = '?' if spec.name is None else spec.name - if spec.origin is None: - if spec.loader is None: - return ''.format(name) - else: - return ''.format(name, spec.loader) - else: - if spec.has_location: - return ''.format(name, spec.origin) - else: - return ''.format(spec.name, spec.origin) +if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? - def init_module_attrs(self, module, _override=False, _force_name=True): - """Set the module's attributes. + import io + import errno + import imp - All missing import-related module attributes will be set. Here - is how the spec attributes map onto the module: + try: + ImportError('msg', name='name', path='path') + except TypeError: + class _ImportError(ImportError): + def __init__(self, *args, **kwargs): + self.name = kwargs.pop('name', None) + self.path = kwargs.pop('path', None) + super(_ImportError, self).__init__(*args, **kwargs) + else: + _ImportError = ImportError - spec.name -> module.__name__ - spec.loader -> module.__loader__ - spec.parent -> module.__package__ - spec -> module.__spec__ + # Frame stripping magic ############################################### - Optional: - spec.origin -> module.__file__ (if spec.set_fileattr is true) - spec.cached -> module.__cached__ (if __file__ also set) - spec.submodule_search_locations -> module.__path__ (if set) + def _call_with_frames_removed(f, *args, **kwds): + """remove_importlib_frames in import.c will always remove sequences + of importlib frames that end with a call to this function - """ - spec = self.spec + Use it instead of a normal call in places where including the importlib + frames introduces unwanted noise into the traceback (e.g. when executing + module code) + """ + return f(*args, **kwds) - # The passed in module may be not support attribute assignment, - # in which case we simply don't set the attributes. + def decode_source(source_bytes): + """Decode bytes representing source code and return the string. - # __name__ - if (_override or _force_name or - getattr(module, '__name__', None) is None): - try: - module.__name__ = spec.name - except AttributeError: - pass - - # __loader__ - if _override or getattr(module, '__loader__', None) is None: - loader = spec.loader - if loader is None: - # A backward compatibility hack. - if spec.submodule_search_locations is not None: - loader = _NamespaceLoader.__new__(_NamespaceLoader) - loader._path = spec.submodule_search_locations - try: - module.__loader__ = loader - except AttributeError: - pass + Universal newline support is used in the decoding. + """ + import tokenize # To avoid bootstrap issues. + source_bytes_readline = io.BytesIO(source_bytes).readline + encoding = tokenize.detect_encoding(source_bytes_readline) + newline_decoder = io.IncrementalNewlineDecoder(None, True) + return newline_decoder.decode(source_bytes.decode(encoding[0])) - # __package__ - if _override or getattr(module, '__package__', None) is None: - try: - module.__package__ = spec.parent - except AttributeError: - pass - # __spec__ - try: - module.__spec__ = spec - except AttributeError: - pass - - # __path__ - if _override or getattr(module, '__path__', None) is None: - if spec.submodule_search_locations is not None: - try: - module.__path__ = spec.submodule_search_locations - except AttributeError: - pass - - if spec.has_location: - # __file__ - if _override or getattr(module, '__file__', None) is None: - try: - module.__file__ = spec.origin - except AttributeError: - pass - - # __cached__ - if _override or getattr(module, '__cached__', None) is None: - if spec.cached is not None: - try: - module.__cached__ = spec.cached - except AttributeError: - pass - - def create(self): - """Return a new module to be loaded. - - The import-related module attributes are also set with the - appropriate values from the spec. + # inspired from importlib2 + class FileLoader2(object): + """Base file loader class which implements the loader protocol methods that + require file system usage. Also implements implicit namespace package PEP 420.""" - """ - spec = self.spec - # Typically loaders will not implement create_module(). - if hasattr(spec.loader, 'create_module'): - # If create_module() returns `None` it means the default - # module creation should be used. - module = spec.loader.create_module(spec) - else: - module = None - if module is None: - # This must be done before open() is ever called as the 'io' - # module implicitly imports 'locale' and would otherwise - # trigger an infinite loop. - module = _new_module(spec.name) - self.init_module_attrs(module) - return module + def __init__(self, fullname, path): + """Cache the module name and the path to the file found by the + finder.""" + self.name = fullname + self.path = path - def _exec(self, module): - """Do everything necessary to execute the module. + def __eq__(self, other): + return (self.__class__ == other.__class__ and + self.__dict__ == other.__dict__) - The namespace of `module` is used as the target of execution. - This method uses the loader's `exec_module()` method. + def __hash__(self): + return hash(self.name) ^ hash(self.path) + def get_source(self, fullname): + """Concrete implementation of InspectLoader.get_source.""" + path = self.get_filename(fullname) + try: + source_bytes = self.get_data(path) + except OSError as exc: + e = _ImportError('source not available through get_data()', + name=fullname) + e.__cause__ = exc + raise e + return source_bytes + # return decode_source(source_bytes) + + def load_module(self, name): + """Load a module from a file. """ - self.spec.loader.exec_module(module) - - # Used by importlib.reload() and _load_module_shim(). - def exec_(self, module): - """Execute the spec in an existing module's namespace.""" - name = self.spec.name - _imp.acquire_lock() - with _ModuleLockManager(name): - if sys.modules.get(name) is not module: - msg = 'module {!r} not in sys.modules'.format(name) - raise _ImportError(msg, name=name) - if self.spec.loader is None: - if self.spec.submodule_search_locations is None: - raise _ImportError('missing loader', name=self.spec.name) - # namespace package - self.init_module_attrs(module, _override=True) - return module - self.init_module_attrs(module, _override=True) - if not hasattr(self.spec.loader, 'exec_module'): - # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # have exec_module() implemented, we can add a deprecation - # warning here. - self.spec.loader.load_module(name) + # Implementation inspired from pytest + + # If there is an existing module object named 'fullname' in + # sys.modules, the loader must use that existing module. (Otherwise, + # the reload() builtin will not work correctly.) + if name in sys.modules: + return sys.modules[name] + + source = self.get_source(name) + # I wish I could just call imp.load_compiled here, but __file__ has to + # be set properly. In Python 3.2+, this all would be handled correctly + # by load_compiled. + mod = sys.modules.setdefault(name, imp.new_module(name)) + try: + # Set a few properties required by PEP 302 + mod.__file__ = self.get_filename(name) + mod.__loader__ = self + if self.is_package(name): + mod.__path__ = [self.path] + mod.__package__ = name # PEP 366 else: - self._exec(module) - return sys.modules[name] + mod.__path__ = self.path + mod.__package__ = '.'.join(name.split('.')[:-1]) # PEP 366 - # def _load_backward_compatible(self): - # # (issue19713) Once BuiltinImporter and ExtensionFileLoader - # # have exec_module() implemented, we can add a deprecation - # # warning here. - # spec = self.spec - # spec.loader.load_module(spec.name) - # # The module must be in sys.modules at this point! - # module = sys.modules[spec.name] - # if getattr(module, '__loader__', None) is None: - # try: - # module.__loader__ = spec.loader - # except AttributeError: - # pass - # if getattr(module, '__package__', None) is None: - # try: - # # Since module.__path__ may not line up with - # # spec.submodule_search_paths, we can't necessarily rely - # # on spec.parent here. - # module.__package__ = module.__name__ - # if not hasattr(module, '__path__'): - # module.__package__ = spec.name.rpartition('.')[0] - # except AttributeError: - # pass - # if getattr(module, '__spec__', None) is None: - # try: - # module.__spec__ = spec - # except AttributeError: - # pass - # return module - - # def _load_unlocked(self): - # # A helper for direct use by the import system. - # if self.spec.loader is not None: - # # not a namespace package - # if not hasattr(self.spec.loader, 'exec_module'): - # return self._load_backward_compatible() - # - # module = self.create() - # with _installed_safely(module): - # if self.spec.loader is None: - # if self.spec.submodule_search_locations is None: - # raise _ImportError('missing loader', name=self.spec.name) - # # A namespace package so do nothing. - # else: - # self._exec(module) - # - # # We don't ensure that the import-related module attributes get - # # set in the sys.modules replacement case. Such modules are on - # # their own. - # return sys.modules[self.spec.name] - - # A method used during testing of _load_unlocked() and by - # _load_module_shim(). - def load(self): - """Return a new module object, loaded by the spec's loader. - - The module is not added to its parent. - - If a module is already in sys.modules, that existing module gets - clobbered. + exec source in mod.__dict__ - """ - _imp.acquire_lock() - with _ModuleLockManager(self.spec.name): - return self._load_unlocked() + except: + if name in sys.modules: + del sys.modules[name] + raise + return sys.modules[name] - def _new_module(name): - return type(sys)(name) + def get_filename(self, fullname): + """Return the path to the source file.""" + if os.path.isdir(self.path) and os.path.isfile(os.path.join(self.path, '__init__.py')): + return os.path.join(self.path, '__init__.py') # python package + else: + return self.path # module or namespace package case - # inspired from importlib2 - class _NamespacePath(object): - """Represents a namespace package's path. It uses the module name - to find its parent module, and from there it looks up the parent's - __path__. When this changes, the module's own path is recomputed, - using path_finder. For top-level modules, the parent module's path - is sys.path.""" - - def __init__(self, name, path, path_finder): - self._name = name - self._path = path - self._last_parent_path = tuple(self._get_parent_path()) - self._path_finder = path_finder - - def _find_parent_path_names(self): - """Returns a tuple of (parent-module-name, parent-path-attr-name)""" - parent, dot, me = self._name.rpartition('.') - if dot == '': - # This is a top-level module. sys.path contains the parent path. - return 'sys', 'path' - # Not a top-level module. parent-module.__path__ contains the - # parent path. - return parent, '__path__' - - def _get_parent_path(self): - parent_module_name, path_attr_name = self._find_parent_path_names() - return getattr(sys.modules[parent_module_name], path_attr_name) - - def _recalculate(self): - # If the parent's path has changed, recalculate _path - parent_path = tuple(self._get_parent_path()) # Make a copy - if parent_path != self._last_parent_path: - spec = self._path_finder(self._name, parent_path) - # Note that no changes are made if a loader is returned, but we - # do remember the new parent path - if spec is not None and spec.loader is None: - if spec.submodule_search_locations: - self._path = spec.submodule_search_locations - self._last_parent_path = parent_path # Save the copy - return self._path - - def __iter__(self): - return iter(self._recalculate()) - - def __len__(self): - return len(self._recalculate()) - - def __repr__(self): - return '_NamespacePath({!r})'.format(self._path) - - def __contains__(self, item): - return item in self._recalculate() - - def append(self, item): - self._path.append(item) + def is_package(self, fullname): + # TODO : test this (without ROS loader) + # in case of package we have to always have the directory as self.path + # we can always compute init path dynamically when needed. + return os.path.isdir(self.path) + + def get_data(self, path): + """Return the data from path as raw bytes. + Implements PEP 420 using pkg_resources + """ + try: + with io.FileIO(path, 'r') as file: + return file.read() + except IOError as ioe: + if ioe.errno == errno.EISDIR: + # implicit namespace package + return """import pkg_resources; pkg_resources.declare_namespace(__name__)""" + else: + raise - # inspired from importlib2 - class _NamespaceLoader(object): - def __init__(self, name, path, subdirs): - self._path = subdirs - @classmethod - def module_repr(cls, module): - """Return repr for the module. +def RosLoader(rosdef_extension): + """ + Function generating ROS loaders. + This is used to keep .msg and .srv loaders very similar + """ + if rosdef_extension == '.msg': + loader_origin_subdir = 'msg' + loader_file_extension = rosdef_extension + loader_generated_subdir = 'msg' + elif rosdef_extension == '.srv': + loader_origin_subdir = 'srv' + loader_file_extension = rosdef_extension + loader_generated_subdir = 'srv' + else: + raise RuntimeError("RosLoader for a format {0} other than .msg or .srv is not supported".format(rosdef_extension)) - The method is deprecated. The import machinery does the job itself. + if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? - """ - return ''.format(module.__name__) + class ROSDefLoader(FileLoader2): - def is_package(self, fullname): - return True + def get_gen_path(self): + """Returning the generated path matching the import""" + return os.path.join(self.outdir_pkg, loader_generated_subdir) - def get_source(self, fullname): - return '' + def __repr__(self): + return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - def get_code(self, fullname): - return compile('', '', 'exec', dont_inherit=True) + def __init__(self, fullname, path, implicit_ns_pkg=False): - def exec_module(self, module): - pass + self.logger = logging.getLogger(__name__) - def _init_module_attrs(self, spec, module, _override=False, _force_name=True): - """Set the module's attributes. + self.implicit_ns_package = implicit_ns_pkg - All missing import-related module attributes will be set. Here - is how the spec attributes map onto the module: + # Doing this in each loader, in case we are running from different processes, + # avoiding to reload from same file (especially useful for boxed tests). + # But deterministic path to avoid regenerating from the same interpreter + self.rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) + if os.path.exists(self.rosimport_path): + shutil.rmtree(self.rosimport_path) + os.makedirs(self.rosimport_path) - spec.name -> module.__name__ - spec.loader -> module.__loader__ - spec.parent -> module.__package__ - spec -> module.__spec__ + self.rospackage = fullname.partition('.')[0] + # We should reproduce package structure in generated file structure + dirlist = path.split(os.sep) + pkgidx = dirlist[::-1].index(self.rospackage) + indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != loader_origin_subdir and not p.endswith(loader_file_extension)] + self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) - Optional: - spec.origin -> module.__file__ (if spec.set_fileattr is true) - spec.cached -> module.__cached__ (if __file__ also set) - spec.submodule_search_locations -> module.__path__ (if set) + # : hack to be able to import a generated class (if requested) + # self.requested_class = None - """ - # The passed in module may be not support attribute assignment, - # in which case we simply don't set the attributes. + if os.path.isdir(path): + if path.endswith(loader_origin_subdir) and any([f.endswith(loader_file_extension) for f in os.listdir(path)]): # if we get a non empty 'msg' folder + init_path = os.path.join(self.outdir_pkg, loader_generated_subdir, '__init__.py') + if not os.path.exists(init_path): + # TODO : we need to determine that from the loader + # as a minimum we need to add current package + self.includepath = [self.rospackage + ':' + path] - # __name__ - if (_override or _force_name or - getattr(module, '__name__', None) is None): - try: - module.__name__ = spec.name - except AttributeError: - pass - - # __loader__ - if _override or getattr(module, '__loader__', None) is None: - loader = spec.loader - if loader is None: - # A backward compatibility hack. - if spec.submodule_search_locations is not None: - loader = _NamespaceLoader.__new__(_NamespaceLoader) - loader._path = spec.submodule_search_locations - try: - module.__loader__ = loader - except AttributeError: - pass + # TODO : unify this after reviewing rosmsg_generator API + if loader_file_extension == '.msg': + # TODO : dynamic in memory generation (we do not need the file ultimately...) + self.gen_msgs = rosmsg_generator.genmsg_py( + msg_files=[os.path.join(path, f) for f in os.listdir(path)], # every file not ending in '.msg' will be ignored + package=self.rospackage, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + initpy=True # we always create an __init__.py when called from here. + ) + init_path = None + for pyf in self.gen_msgs: + if pyf.endswith('__init__.py'): + init_path = pyf + elif loader_file_extension == '.srv': + # TODO : dynamic in memory generation (we do not need the file ultimately...) + self.gen_msgs = rosmsg_generator.gensrv_py( + srv_files=[os.path.join(path, f) for f in os.listdir(path)], + # every file not ending in '.msg' will be ignored + package=self.rospackage, + outdir_pkg=self.outdir_pkg, + includepath=self.includepath, + initpy=True # we always create an __init__.py when called from here. + ) + init_path = None + for pyf in self.gen_msgs: + if pyf.endswith('__init__.py'): + init_path = pyf + else: + raise RuntimeError( + "RosDefLoader for a format {0} other than .msg or .srv is not supported".format( + rosdef_extension)) - # __package__ - if _override or getattr(module, '__package__', None) is None: - try: - module.__package__ = spec.parent - except AttributeError: - pass + if not init_path: + raise ImportError("__init__.py file not found".format(init_path)) + if not os.path.exists(init_path): + raise ImportError("{0} file not found".format(init_path)) - # __spec__ - try: - module.__spec__ = spec - except AttributeError: - pass - - # __path__ - if _override or getattr(module, '__path__', None) is None: - if spec.submodule_search_locations is not None: - try: - module.__path__ = spec.submodule_search_locations - except AttributeError: - pass - - if spec.has_location: - # __file__ - if _override or getattr(module, '__file__', None) is None: - try: - module.__file__ = spec.origin - except AttributeError: - pass - - # __cached__ - if _override or getattr(module, '__cached__', None) is None: - if spec.cached is not None: - try: - module.__cached__ = spec.cached - except AttributeError: - pass - - def _create(self, spec): - """Return a new module to be loaded. - - The import-related module attributes are also set with the - appropriate values from the spec. + # relying on usual source file loader since we have generated normal python code + super(ROSDefLoader, self).__init__(fullname, init_path) + else: # it is a directory potentially containing an 'msg' + # If we are here, it means it wasn't loaded before + # We need to be able to load from source + super(ROSDefLoader, self).__init__(fullname, path) - """ - # Typically loaders will not implement create_module(). - if hasattr(spec.loader, 'create_module'): - # If create_module() returns `None` it means the default - # module creation should be used. - module = spec.loader.create_module(spec) - else: - module = None - if module is None: - # This must be done before open() is ever called as the 'io' - # module implicitly imports 'locale' and would otherwise - # trigger an infinite loop. - module = _new_module(spec.name) - self._init_module_attrs(spec, module) - return module + # or to load from installed ros package (python already generated, no point to generate again) + # Note : the path being in sys.path or not is a matter of ROS setup or metafinder. + # TODO - def load_module(self, fullname): - """Load a namespace module. + elif os.path.isfile(path): + # The file should have already been generated (by the loader for a msg package) + # Note we do not want to rely on namespace packages here, since they are not standardized for python2, + # and they can prevent some useful usecases. - This method is deprecated. Use exec_module() instead. + # Hack to be able to "import generated classes" + modname = fullname.rpartition('.')[2] + filepath = os.path.join(self.outdir_pkg, loader_generated_subdir, '_' + modname + '.py') # the generated module + # relying on usual source file loader since we have previously generated normal python code + super(ROSDefLoader, self).__init__(fullname, filepath) - """ - # The import system never calls this method. - _verbose_message('namespace module loaded with path {!r}', self._path) - - spec = importlib_util.spec_from_loader(fullname, self) - spec.submodule_search_locations = self._path - if fullname in sys.modules: - module = sys.modules[fullname] - self._init_module_attrs(spec, module, _override=True) - return sys.modules[fullname] - else: - module = self._create(spec) - return module + def get_gen_path(self): + """Returning the generated path matching the import""" + return os.path.join(self.outdir_pkg, loader_generated_subdir) + def is_package(self, fullname): + return self.implicit_ns_package or super(ROSDefLoader, self).is_package(fullname=fullname) -elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 but importlib2 theoretically could... - import importlib.machinery as importlib_machinery + def get_source(self, fullname): + """Concrete implementation of InspectLoader.get_source.""" + path = self.get_filename(fullname) + try: + source_bytes = self.get_data(path) + except OSError as exc: + e = _ImportError('source not available through get_data()', + name=fullname) + e.__cause__ = exc + raise e + return source_bytes + # return decode_source(source_bytes) + + def source_to_code(self, data, path, _optimize=-1): + """Return the code object compiled from source. + + The 'data' argument can be any object type that compile() supports. + """ + # XXX Handle _optimize when possible? + return _call_with_frames_removed(compile, data, path, 'exec', + dont_inherit=True) + def get_code(self, fullname): + source = self.get_source(fullname) + print('compiling code for "%s"' % fullname) + return compile(source, self._get_filename(fullname), 'exec', dont_inherit=True) + + def __repr__(self): + return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) + + def load_module(self, fullname): + source = self.get_source(fullname) + + if fullname in sys.modules: + print('reusing existing module from previous import of "%s"' % fullname) + mod = sys.modules[fullname] + else: + print('creating a new module object for "%s"' % fullname) + mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) -else: - raise ImportError("ros_loader : Unsupported python version") + # Set a few properties required by PEP 302 or PEP 420 + if self.implicit_ns_package: + mod.__path__ = [self.path] + else: + mod.__file__ = self.get_filename(fullname) + mod.__path__ = self.path + + mod.__name__ = fullname + mod.__loader__ = self + mod.__package__ = '.'.join(fullname.split('.')[:-1]) + + if self.is_package(fullname): + print('adding path for package') + # Set __path__ for packages + # so we can find the sub-modules. + mod.__path__ = [self.path] + else: + print('imported as regular module') + # Note : we want to avoid calling imp.load_module, + # to eventually be able to implement this without generating temporary file + print('execing source...') + exec source in mod.__dict__ + print('done') + return mod -def RosLoader(rosdef_extension): - """ - Function generating ROS loaders. - This is used to keep .msg and .srv loaders very similar - """ - if rosdef_extension == '.msg': - loader_origin_subdir = 'msg' - loader_file_extension = rosdef_extension - loader_generated_subdir = 'msg' - elif rosdef_extension == '.srv': - loader_origin_subdir = 'srv' - loader_file_extension = rosdef_extension - loader_generated_subdir = 'srv' - else: - raise RuntimeError("RosLoader for a format {0} other than .msg or .srv is not supported".format(rosdef_extension)) + elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 (unsupported but might work ?) + import importlib.machinery as importlib_machinery - class ROSDefLoader(importlib_machinery.SourceFileLoader): - def __init__(self, fullname, path, implicit_ns_pkg=False): + class ROSDefLoader(importlib_machinery.SourceFileLoader): + def __init__(self, fullname, path, implicit_ns_pkg=False): - self.logger = logging.getLogger(__name__) + self.logger = logging.getLogger(__name__) - self.implicit_ns_package = implicit_ns_pkg + self.implicit_ns_package = implicit_ns_pkg - if (2, 7) <= sys.version_info < (3, 4) and implicit_ns_pkg: # the specific python2 usecase where we want a directory to be considered as a namespace package - # we need to use the usual FileFinder logic - super(ROSDefLoader, self).__init__(fullname, path) - else: # Doing this in each loader, in case we are running from different processes, # avoiding to reload from same file (especially useful for boxed tests). # But deterministic path to avoid regenerating from the same interpreter @@ -663,21 +461,24 @@ def __init__(self, fullname, path, implicit_ns_pkg=False): # relying on usual source file loader since we have previously generated normal python code super(ROSDefLoader, self).__init__(fullname, filepath) - def get_gen_path(self): - """Returning the generated path matching the import""" - return os.path.join(self.outdir_pkg, loader_generated_subdir) + def get_gen_path(self): + """Returning the generated path matching the import""" + return os.path.join(self.outdir_pkg, loader_generated_subdir) - def __repr__(self): - return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) + def __repr__(self): + return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - def exec_module(self, module): - # Custom implementation to declare an implicit namespace package - if (2, 7) <= sys.version_info < (3, 4) and self.implicit_ns_package: - module.__path__ = [self.path] # mandatory to trick importlib2 to think this is a package - import pkg_resources - pkg_resources.declare_namespace(module.__name__) - else: - super(ROSDefLoader, self).exec_module(module) + def exec_module(self, module): + # Custom implementation to declare an implicit namespace package + if (2, 7) <= sys.version_info < (3, 4) and self.implicit_ns_package: + module.__path__ = [self.path] # mandatory to trick importlib2 to think this is a package + import pkg_resources + pkg_resources.declare_namespace(module.__name__) + else: + super(ROSDefLoader, self).exec_module(module) + + else: + raise ImportError("ros_loader : Unsupported python version") return ROSDefLoader diff --git a/pyros_msgs/importer/tests/nspkg/subpkg/__init__.py b/pyros_msgs/importer/tests/nspkg/subpkg/__init__.py new file mode 100644 index 0000000..df5fa9d --- /dev/null +++ b/pyros_msgs/importer/tests/nspkg/subpkg/__init__.py @@ -0,0 +1,7 @@ +from __future__ import absolute_import + +from . import submodule + +# Just Dummy class for testing +class TestClassInSubPkg: + pass diff --git a/pyros_msgs/importer/tests/nspkg/subpkg/submodule.py b/pyros_msgs/importer/tests/nspkg/subpkg/submodule.py new file mode 100644 index 0000000..43b1032 --- /dev/null +++ b/pyros_msgs/importer/tests/nspkg/subpkg/submodule.py @@ -0,0 +1,4 @@ + +# Just Dummy class for testing +class TestClassInSubModule: + pass \ No newline at end of file diff --git a/pyros_msgs/importer/tests/test_rosmsg_import.py b/pyros_msgs/importer/tests/test_rosmsg_import.py index 27cac8d..532a339 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_import.py @@ -65,6 +65,63 @@ def print_importers(): print('%s: %r' % (name, cache_value)) +# We need to test implicit namespace packages PEP 420 (especially for python 2.7) +# Since we rely on it for ros import. +# But we can only teet relative package structure +class TestImplicitNamespace(unittest.TestCase): + @classmethod + def setUpClass(cls): + # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package + supported_loaders = rosmsg_finder._get_supported_ns_loaders() + ns_hook = rosmsg_finder.FileFinder2.path_hook(*supported_loaders) + sys.path_hooks.insert(1, ns_hook) + + def test_import_relative_ns_subpkg(self): + """Verify that package is importable relatively""" + print_importers() + + from .nspkg import subpkg as test_pkg + + self.assertTrue(test_pkg is not None) + self.assertTrue(test_pkg.TestClassInSubPkg is not None) + self.assertTrue(callable(test_pkg.TestClassInSubPkg)) + + def test_import_relative_ns_subpkg_submodule(self): + """Verify that package is importable relatively""" + print_importers() + + from .nspkg.subpkg import submodule as test_mod + + self.assertTrue(test_mod is not None) + self.assertTrue(test_mod.TestClassInSubModule is not None) + self.assertTrue(callable(test_mod.TestClassInSubModule)) + + def test_import_class_from_relative_ns_subpkg(self): + """Verify that message class is importable relatively""" + print_importers() + + from .nspkg.subpkg import TestClassInSubPkg + + self.assertTrue(TestClassInSubPkg is not None) + self.assertTrue(callable(TestClassInSubPkg)) + + def test_import_class_from_relative_ns_subpkg_submodule(self): + """Verify that package is importable relatively""" + print_importers() + + from .nspkg.subpkg.submodule import TestClassInSubModule + + self.assertTrue(TestClassInSubModule is not None) + self.assertTrue(callable(TestClassInSubModule)) + + def test_import_relative_nonnspkg_raises(self): + """Verify that package is importable relatively""" + print_importers() + + with self.assertRaises(ImportError): + from .bad_nspkg import bad_subpkg + + class TestImportMsg(unittest.TestCase): rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') From 8575e207c70dfdf5d32f65a344fe0631391e52b1 Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 27 Jun 2017 15:33:49 +0900 Subject: [PATCH 15/28] python2 generated modules import working! --- pyros_msgs/importer/rosmsg_finder.py | 391 +++++------------- pyros_msgs/importer/rosmsg_loader.py | 192 ++++----- .../importer/tests/test_rosmsg_import.py | 14 +- .../importer/tests/test_rosmsg_importlib.py | 2 +- 4 files changed, 176 insertions(+), 423 deletions(-) diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py index 84fa8ea..47818c8 100644 --- a/pyros_msgs/importer/rosmsg_finder.py +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -34,69 +34,69 @@ import imp - def spec_from_file_location(name, location=None, loader=None, - submodule_search_locations=None): - """Return a module spec based on a file location. - - To indicate that the module is a package, set - submodule_search_locations to a list of directory paths. An - empty list is sufficient, though its not otherwise useful to the - import system. - - The loader must take a spec as its only __init__() arg. - - """ - - if location is None: - # The caller may simply want a partially populated location- - # oriented spec. So we set the location to a bogus value and - # fill in as much as we can. - location = '' - if hasattr(loader, 'get_filename'): - # ExecutionLoader - try: - location = loader.get_filename(name) - except ImportError: - pass - - # If the location is on the filesystem, but doesn't actually exist, - # we could return None here, indicating that the location is not - # valid. However, we don't have a good way of testing since an - # indirect location (e.g. a zip file or URL) will look like a - # non-existent file relative to the filesystem. - - spec = ModuleSpec(name, loader, origin=location) - spec._set_fileattr = True - - # Pick a loader if one wasn't provided. - if loader is None: - for loader_class, suffixes in _get_supported_file_loaders(): - if location.endswith(tuple(suffixes)): - loader = loader_class(name, location) - spec.loader = loader - break - else: - return None - - # Set submodule_search_paths appropriately. - if submodule_search_locations is None: - # Check the loader. - if hasattr(loader, 'is_package'): - try: - is_package = loader.is_package(name) - except ImportError: - pass - else: - if is_package: - spec.submodule_search_locations = [] - else: - spec.submodule_search_locations = submodule_search_locations - if spec.submodule_search_locations == []: - if location: - dirname = os.path.split(location)[0] - spec.submodule_search_locations.append(dirname) - - return spec + # def spec_from_file_location(name, location=None, loader=None, + # submodule_search_locations=None): + # """Return a module spec based on a file location. + # + # To indicate that the module is a package, set + # submodule_search_locations to a list of directory paths. An + # empty list is sufficient, though its not otherwise useful to the + # import system. + # + # The loader must take a spec as its only __init__() arg. + # + # """ + # + # if location is None: + # # The caller may simply want a partially populated location- + # # oriented spec. So we set the location to a bogus value and + # # fill in as much as we can. + # location = '' + # if hasattr(loader, 'get_filename'): + # # ExecutionLoader + # try: + # location = loader.get_filename(name) + # except ImportError: + # pass + # + # # If the location is on the filesystem, but doesn't actually exist, + # # we could return None here, indicating that the location is not + # # valid. However, we don't have a good way of testing since an + # # indirect location (e.g. a zip file or URL) will look like a + # # non-existent file relative to the filesystem. + # + # spec = ModuleSpec(name, loader, origin=location) + # spec._set_fileattr = True + # + # # Pick a loader if one wasn't provided. + # if loader is None: + # for loader_class, suffixes in _get_supported_file_loaders(): + # if location.endswith(tuple(suffixes)): + # loader = loader_class(name, location) + # spec.loader = loader + # break + # else: + # return None + # + # # Set submodule_search_paths appropriately. + # if submodule_search_locations is None: + # # Check the loader. + # if hasattr(loader, 'is_package'): + # try: + # is_package = loader.is_package(name) + # except ImportError: + # pass + # else: + # if is_package: + # spec.submodule_search_locations = [] + # else: + # spec.submodule_search_locations = submodule_search_locations + # if spec.submodule_search_locations == []: + # if location: + # dirname = os.path.split(location)[0] + # spec.submodule_search_locations.append(dirname) + # + # return spec class FileFinder2(object): def __init__(self, path, *loader_details): @@ -111,10 +111,6 @@ def __init__(self, path, *loader_details): self.path = path or '.' # Note : we are not playing with cache here (too complex to get right and not worth it for obsolete python) - def _get_spec(self, loader_class, fullname, path, smsl, target): - loader = loader_class(fullname, path) - return spec_from_file_location(fullname, path, loader=loader, submodule_search_locations=smsl) - def find_module(self, fullname, path=None): """Try to find a loader for the specified module, or the namespace package portions. Returns loader.""" @@ -135,21 +131,6 @@ def find_module(self, fullname, path=None): _verbose_message('possible namespace for {}'.format(base_path)) return FileLoader2(fullname, base_path) - # # Check for a file w/ a proper suffix exists. - # for suffix, loader_class in self._loaders: - # full_path = os.path.join(self.path, tail_module + suffix) - # _verbose_message('trying {}'.format(full_path), verbosity=2) - # if cache_module + suffix in cache: - # if os.path.isfile(full_path): - # return loader_class(fullname, full_path) - # #return self._get_spec(loader_class, fullname, full_path, None, target) - - # if is_namespace: - # _verbose_message('possible namespace for {}'.format(base_path)) - # return loader_class(fullname, None) - # # spec = ModuleSpec(fullname, None) - # # spec.submodule_search_locations = [base_path] - # # return spec return None @classmethod @@ -162,107 +143,17 @@ def path_hook(cls, *loader_details): raised. """ - def path_hook_for_FileFinder(path): - """Path hook for importlib.machinery.FileFinder.""" + def path_hook_for_FileFinder2(path): + """Path hook for FileFinder2.""" if not os.path.isdir(path): raise _ImportError('only directories are supported', path=path) return cls(path, *loader_details) - return path_hook_for_FileFinder + return path_hook_for_FileFinder2 def __repr__(self): return 'FileFinder2({!r})'.format(self.path) - - class FileFinder(object): - - """File-based finder. - - Interactions with the file system are cached for performance, being - refreshed when the directory the finder is handling has been modified. - - """ - - def __init__(self, path, *loader_details): - """Initialize with the path to search on and a variable number of - 2-tuples containing the loader and the file suffixes the loader - recognizes.""" - loaders = [] - for loader, suffixes in loader_details: - loaders.extend((suffix, loader) for suffix in suffixes) - self._loaders = loaders - # Base (directory) path - self.path = path or '.' - self._path_mtime = -1 - self._path_cache = set() - self._relaxed_path_cache = set() - - def invalidate_caches(self): - """Invalidate the directory mtime.""" - self._path_mtime = -1 - - #find_module = _find_module_shim - - def find_loader(self, fullname): - """Try to find a loader for the specified module, or the namespace - package portions. Returns (loader, list-of-portions). - - This method is deprecated. Use find_spec() instead. - - """ - spec = self.find_spec(fullname) - if spec is None: - return None, [] - return spec.loader, spec.submodule_search_locations or [] - - - - def find_spec(self, fullname, target=None): - """Try to find a loader for the specified module, or the namespace - package portions. Returns (loader, list-of-portions).""" - is_namespace = False - tail_module = fullname.rpartition('.')[2] - try: - mtime = _path_stat(self.path or _os.getcwd()).st_mtime - except OSError: - mtime = -1 - if mtime != self._path_mtime: - self._fill_cache() - self._path_mtime = mtime - # tail_module keeps the original casing, for __file__ and friends - if _relax_case(): - cache = self._relaxed_path_cache - cache_module = tail_module.lower() - else: - cache = self._path_cache - cache_module = tail_module - # Check if the module is the name of a directory (and thus a package). - if cache_module in cache: - base_path = _path_join(self.path, tail_module) - for suffix, loader_class in self._loaders: - init_filename = '__init__' + suffix - full_path = _path_join(base_path, init_filename) - if _path_isfile(full_path): - return self._get_spec(loader_class, fullname, full_path, [base_path], target) - else: - # If a namespace package, return the path if we don't - # find a module in the next section. - is_namespace = _path_isdir(base_path) - # Check for a file w/ a proper suffix exists. - for suffix, loader_class in self._loaders: - full_path = _path_join(self.path, tail_module + suffix) - _verbose_message('trying {}'.format(full_path), verbosity=2) - if cache_module + suffix in cache: - if _path_isfile(full_path): - return self._get_spec(loader_class, fullname, full_path, None, target) - if is_namespace: - _verbose_message('possible namespace for {}'.format(base_path)) - spec = ModuleSpec(fullname, None) - spec.submodule_search_locations = [base_path] - return spec - return None - - def _get_supported_ns_loaders(): """Returns a list of file-based module loaders. Each item is a tuple (loader, suffixes). @@ -271,7 +162,7 @@ def _get_supported_ns_loaders(): return [loader] - class DirectoryFinder(FileFinder2): + class ROSDirectoryFinder(FileFinder2): """Finder to interpret directories as modules, and files as classes""" def __init__(self, path, *ros_loader_details): @@ -291,32 +182,21 @@ def __init__(self, path, *ros_loader_details): ros_loaders.extend((suffix, loader) for suffix in suffixes) self._ros_loaders = ros_loaders - # # Form importlib FileFinder - # loaders = [] - # for loader, suffixes in [(importlib_machinery.SourceFileLoader, ['.py']), (importlib_machinery.SourcelessFileLoader, ['.pyc'])]: - # loaders.extend((suffix, loader) for suffix in suffixes) - # self._loaders = loaders - # Base (directory) path - self.path = path or '.' - self._path_mtime = -1 - self._path_cache = set() - self._relaxed_path_cache = set() + # We rely on FileFinder and python loader to deal with our generated code + super(ROSDirectoryFinder, self).__init__( + path, + *_get_supported_ns_loaders() + ) def __repr__(self): - return 'DirectoryFinder({!r})'.format(self.path) + return 'ROSDirectoryFinder({!r})'.format(self.path) def find_module(self, fullname, path=None): path = path or self.path - tail_module = fullname.rpartition('.')[2] - loader = None - # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. - # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package - # tail_module = tail_module[1:] - base_path = os.path.join(self.path, tail_module) - + base_path = os.path.join(path, tail_module) # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. if os.path.isdir(base_path): loader_class = None @@ -327,107 +207,34 @@ def find_module(self, fullname, path=None): if any(f.endswith(suffix) for f in files): loader_class = loader_cls rosdir = root - if loader_class and rosdir: - if rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module - # Generate something ! - # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location - loader = loader_class(fullname, base_path) # loader.get_gen_path()]) - # We DO NOT WANT TO add the generated dir in sys.path to use a python loader - # since the plan is to eventually not have to rely on files at all TODO - else: - # directory not matching our path, so this should be a namespace, and has to be explicit in python < 3.2 - - # But we still rely on importlib2 code to backport namespace packages logic - # Done via loader custom code - loader = loader_class(fullname, base_path, implicit_ns_pkg=True) - - # we return None if we couldn't build a loader before + if loader_class and rosdir and rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module + # Generate something ! + # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location + loader = loader_class(fullname, base_path) # loader.get_gen_path()]) + # We DO NOT WANT TO add the generated dir in sys.path to use a python loader + # since the plan is to eventually not have to rely on files at all TODO + + # if we couldn't build a loader before we forward the call to our parent FileFinder2 (useful for implicit namespace packages) + loader = loader or super(ROSDirectoryFinder, self).find_module(fullname, path) + # If we couldnt find any loader before, we return None return loader - # inspired from importlib2 @classmethod def path_hook(cls, *loader_details): - """A class method which returns a closure to use on sys.path_hook - which will return an instance using the specified loaders and the path - called on the closure. - - If the path called on the closure is not a directory, ImportError is - raised. - - """ - - def path_hook_for_FileFinder(path): - """Path hook for rosimport.DirectoryFinder.""" + # Same as FileFinder2 + def path_hook_for_ROSDirectoryFinder(path): + """Path hook for ROSDirectoryFinder.""" if not os.path.isdir(path): raise _ImportError('only directories are supported', path=path) return cls(path, *loader_details) - return path_hook_for_FileFinder + return path_hook_for_ROSDirectoryFinder - def __repr__(self): - return 'FileFinder({!r})'.format(self.path) - - # def find_spec(self, fullname, target=None): - # """ - # Try to find a spec for the specified module. - # Returns the matching spec, or None if not found. - # :param fullname: the name of the package we are trying to import - # :param target: what we plan to do with it - # :return: - # """ - # - # tail_module = fullname.rpartition('.')[2] - # - # spec = None - # - # # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. - # # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package - # # tail_module = tail_module[1:] - # base_path = os.path.join(self.path, tail_module) - # - # # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. - # if os.path.isdir(base_path): - # loader_class = None - # rosdir = None - # # Figuring out if we should care about this directory at all - # for root, dirs, files in os.walk(base_path): - # for suffix, loader_cls in self._ros_loaders: - # if any(f.endswith(suffix) for f in files): - # loader_class = loader_cls - # rosdir = root - # if loader_class and rosdir: - # if rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module - # # Generate something ! - # loader = loader_class(fullname, base_path) - # # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location - # spec = importlib_util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) - # # We DO NOT WANT TO add the generated dir in sys.path to use a python loader - # # since the plan is to eventually not have to rely on files at all TODO - # # elif (2, 7) <= sys.version_info < (3, 4): - # # # Found nothing, so this should be a namespace, and has to be explicit in python < 3.2 - # # # But we still rely on importlib2 code to backport namespace packages logic - # # # Done via loader custom code - # # spec = importlib_util.spec_from_file_location(fullname, base_path, - # # loader=None, - # # #loader=loader_class(fullname, base_path, implicit_ns_pkg=True) - # # submodule_search_locations=[base_path]) - # # #spec = importlib_machinery.ModuleSpec(name=fullname, origin=base_path, loader=loader_class(fullname, base_path, implicit_ns_pkg=True), is_package=True) - # - # # Relying on FileFinder if we couldn't find any specific directory structure/content - # # It will return a namespace spec if no file can be found (even with python2.7, thanks to importlib2) - # # or will return a proper loader for already generated python files - # spec = spec or super(DirectoryFinder, self).find_spec(fullname, target=target) - # # we return None if we couldn't find a spec before - # return spec - - - -elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 but importlib2 theoritically could... +elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 (maybe we could, if it was worth it...) import importlib.machinery as importlib_machinery import importlib.util as importlib_util - class DirectoryFinder(importlib_machinery.FileFinder): """Finder to interpret directories as modules, and files as classes""" @@ -449,7 +256,7 @@ def __init__(self, path, *ros_loader_details): self._ros_loaders = ros_loaders # We rely on FileFinder and python loader to deal with our generated code - super(DirectoryFinder, self).__init__( + super(ROSDirectoryFinder, self).__init__( path, (importlib_machinery.SourceFileLoader, ['.py']), (importlib_machinery.SourcelessFileLoader, ['.pyc']), @@ -468,12 +275,7 @@ def find_spec(self, fullname, target=None): """ tail_module = fullname.rpartition('.')[2] - spec = None - - # ignoring the generated message module when looking for it. Instead we want to start again from the original .msg/.srv. - # if tail_module.startswith('_') and any(os.path.isfile(os.path.join(self.path, tail_module[1:] + sfx) for sfx, _ in self._loaders)): # consistent with generated module from msg package - # tail_module = tail_module[1:] base_path = os.path.join(self.path, tail_module) # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. @@ -494,20 +296,11 @@ def find_spec(self, fullname, target=None): spec = importlib_util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) # We DO NOT WANT TO add the generated dir in sys.path to use a python loader # since the plan is to eventually not have to rely on files at all TODO - # elif (2, 7) <= sys.version_info < (3, 4): - # # Found nothing, so this should be a namespace, and has to be explicit in python < 3.2 - # # But we still rely on importlib2 code to backport namespace packages logic - # # Done via loader custom code - # spec = importlib_util.spec_from_file_location(fullname, base_path, - # loader=None, - # #loader=loader_class(fullname, base_path, implicit_ns_pkg=True) - # submodule_search_locations=[base_path]) - # #spec = importlib_machinery.ModuleSpec(name=fullname, origin=base_path, loader=loader_class(fullname, base_path, implicit_ns_pkg=True), is_package=True) # Relying on FileFinder if we couldn't find any specific directory structure/content # It will return a namespace spec if no file can be found (even with python2.7, thanks to importlib2) # or will return a proper loader for already generated python files - spec = spec or super(DirectoryFinder, self).find_spec(fullname, target=target) + spec = spec or super(ROSDirectoryFinder, self).find_spec(fullname, target=target) # we return None if we couldn't find a spec before return spec @@ -531,7 +324,7 @@ def _get_supported_ros_loaders(): def _install(): """Install the path-based import components.""" supported_loaders = _get_supported_ros_loaders() - sys.path_hooks.extend([DirectoryFinder.path_hook(*supported_loaders)]) + sys.path_hooks.extend([ROSDirectoryFinder.path_hook(*supported_loaders)]) # TODO : sys.meta_path.append(DistroFinder) diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index 2c6fd71..d6708d3 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -58,38 +58,48 @@ def __init__(self, *args, **kwargs): # Frame stripping magic ############################################### - def _call_with_frames_removed(f, *args, **kwds): - """remove_importlib_frames in import.c will always remove sequences - of importlib frames that end with a call to this function - - Use it instead of a normal call in places where including the importlib - frames introduces unwanted noise into the traceback (e.g. when executing - module code) - """ - return f(*args, **kwds) - - def decode_source(source_bytes): - """Decode bytes representing source code and return the string. - - Universal newline support is used in the decoding. - """ - import tokenize # To avoid bootstrap issues. - source_bytes_readline = io.BytesIO(source_bytes).readline - encoding = tokenize.detect_encoding(source_bytes_readline) - newline_decoder = io.IncrementalNewlineDecoder(None, True) - return newline_decoder.decode(source_bytes.decode(encoding[0])) - + # def _call_with_frames_removed(f, *args, **kwds): + # """remove_importlib_frames in import.c will always remove sequences + # of importlib frames that end with a call to this function + # + # Use it instead of a normal call in places where including the importlib + # frames introduces unwanted noise into the traceback (e.g. when executing + # module code) + # """ + # return f(*args, **kwds) + # + # def decode_source(source_bytes): + # """Decode bytes representing source code and return the string. + # + # Universal newline support is used in the decoding. + # """ + # import tokenize # To avoid bootstrap issues. + # source_bytes_readline = io.BytesIO(source_bytes).readline + # encoding = tokenize.detect_encoding(source_bytes_readline) + # newline_decoder = io.IncrementalNewlineDecoder(None, True) + # return newline_decoder.decode(source_bytes.decode(encoding[0])) # inspired from importlib2 class FileLoader2(object): """Base file loader class which implements the loader protocol methods that - require file system usage. Also implements implicit namespace package PEP 420.""" + require file system usage. Also implements implicit namespace package PEP 420. + + CAREFUL: the finder/loader logic is DIFFERENT than for python3. + A namespace package, or a normal package, need to return a directory path. + get_filename() will append '__init__.py' if it exists and needs to be called when a module path is needed + """ def __init__(self, fullname, path): """Cache the module name and the path to the file found by the - finder.""" + finder. + :param fullname the name of the module to load + :param path to the module or the package. + If it is a package (namespace or not) the path must point to a directory. + Otherwise the path to the python module is needed + """ self.name = fullname self.path = path + # Note for namespace package, we let pkg_resources manage the __path__ logic def __eq__(self, other): return (self.__class__ == other.__class__ and @@ -114,15 +124,18 @@ def get_source(self, fullname): def load_module(self, name): """Load a module from a file. """ - # Implementation inspired from pytest + # Implementation inspired from pytest.rewrite and importlib - # If there is an existing module object named 'fullname' in + # If there is an existing module object named 'name' in # sys.modules, the loader must use that existing module. (Otherwise, # the reload() builtin will not work correctly.) if name in sys.modules: return sys.modules[name] - source = self.get_source(name) + code = self.get_code(name) + if code is None: + raise ImportError('cannot load module {!r} when get_code() ' + 'returns None'.format(name)) # I wish I could just call imp.load_compiled here, but __file__ has to # be set properly. In Python 3.2+, this all would be handled correctly # by load_compiled. @@ -135,10 +148,11 @@ def load_module(self, name): mod.__path__ = [self.path] mod.__package__ = name # PEP 366 else: - mod.__path__ = self.path mod.__package__ = '.'.join(name.split('.')[:-1]) # PEP 366 - - exec source in mod.__dict__ + # if we want to skip compilation - useful for debugging + source = self.get_source(name) + exec (source, mod.__dict__) + #exec(code, mod.__dict__) except: if name in sys.modules: @@ -153,6 +167,11 @@ def get_filename(self, fullname): else: return self.path # module or namespace package case + def get_code(self, fullname): + source = self.get_source(fullname) + print('compiling code for "%s"' % fullname) + return compile(source, self.get_filename(fullname), 'exec', dont_inherit=True) + def is_package(self, fullname): # TODO : test this (without ROS loader) # in case of package we have to always have the directory as self.path @@ -201,11 +220,16 @@ def get_gen_path(self): def __repr__(self): return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - def __init__(self, fullname, path, implicit_ns_pkg=False): + # def get_filename(self, fullname): + # """Return the path to the source file.""" + # if os.path.isdir(self.path) and os.path.isfile(os.path.join(self.path, '__init__.py')): + # return os.path.join(self.path, '__init__.py') # python package + # else: + # return self.path # module or namespace package case - self.logger = logging.getLogger(__name__) + def __init__(self, fullname, path): - self.implicit_ns_package = implicit_ns_pkg + self.logger = logging.getLogger(__name__) # Doing this in each loader, in case we are running from different processes, # avoiding to reload from same file (especially useful for boxed tests). @@ -231,7 +255,7 @@ def __init__(self, fullname, path, implicit_ns_pkg=False): if not os.path.exists(init_path): # TODO : we need to determine that from the loader # as a minimum we need to add current package - self.includepath = [self.rospackage + ':' + path] + self.includepath = [self.rospackage + ':' + path] # TODO :maybe keep a list of all messages we have imported (sys.rosmessages) # TODO : unify this after reviewing rosmsg_generator API if loader_file_extension == '.msg': @@ -272,7 +296,8 @@ def __init__(self, fullname, path, implicit_ns_pkg=False): raise ImportError("{0} file not found".format(init_path)) # relying on usual source file loader since we have generated normal python code - super(ROSDefLoader, self).__init__(fullname, init_path) + # BUT we need to pass the directory path (not the init file path like for python3) + super(ROSDefLoader, self).__init__(fullname, os.path.dirname(init_path)) else: # it is a directory potentially containing an 'msg' # If we are here, it means it wasn't loaded before # We need to be able to load from source @@ -292,89 +317,22 @@ def __init__(self, fullname, path, implicit_ns_pkg=False): filepath = os.path.join(self.outdir_pkg, loader_generated_subdir, '_' + modname + '.py') # the generated module # relying on usual source file loader since we have previously generated normal python code super(ROSDefLoader, self).__init__(fullname, filepath) - - def get_gen_path(self): - """Returning the generated path matching the import""" - return os.path.join(self.outdir_pkg, loader_generated_subdir) - - def is_package(self, fullname): - return self.implicit_ns_package or super(ROSDefLoader, self).is_package(fullname=fullname) - - def get_source(self, fullname): - """Concrete implementation of InspectLoader.get_source.""" - path = self.get_filename(fullname) - try: - source_bytes = self.get_data(path) - except OSError as exc: - e = _ImportError('source not available through get_data()', - name=fullname) - e.__cause__ = exc - raise e - return source_bytes - # return decode_source(source_bytes) - - def source_to_code(self, data, path, _optimize=-1): - """Return the code object compiled from source. - - The 'data' argument can be any object type that compile() supports. - """ - # XXX Handle _optimize when possible? - return _call_with_frames_removed(compile, data, path, 'exec', - dont_inherit=True) - def get_code(self, fullname): - source = self.get_source(fullname) - print('compiling code for "%s"' % fullname) - return compile(source, self._get_filename(fullname), 'exec', dont_inherit=True) - - def __repr__(self): - return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - - def load_module(self, fullname): - source = self.get_source(fullname) - - if fullname in sys.modules: - print('reusing existing module from previous import of "%s"' % fullname) - mod = sys.modules[fullname] - else: - print('creating a new module object for "%s"' % fullname) - mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) - - # Set a few properties required by PEP 302 or PEP 420 - if self.implicit_ns_package: - mod.__path__ = [self.path] - else: - mod.__file__ = self.get_filename(fullname) - mod.__path__ = self.path - - mod.__name__ = fullname - mod.__loader__ = self - mod.__package__ = '.'.join(fullname.split('.')[:-1]) - - if self.is_package(fullname): - print('adding path for package') - # Set __path__ for packages - # so we can find the sub-modules. - mod.__path__ = [self.path] - else: - print('imported as regular module') - - # Note : we want to avoid calling imp.load_module, - # to eventually be able to implement this without generating temporary file - print('execing source...') - exec source in mod.__dict__ - print('done') - return mod + # + # def get_gen_path(self): + # """Returning the generated path matching the import""" + # return os.path.join(self.outdir_pkg, loader_generated_subdir) + # + # def __repr__(self): + # return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 (unsupported but might work ?) import importlib.machinery as importlib_machinery class ROSDefLoader(importlib_machinery.SourceFileLoader): - def __init__(self, fullname, path, implicit_ns_pkg=False): + def __init__(self, fullname, path): self.logger = logging.getLogger(__name__) - self.implicit_ns_package = implicit_ns_pkg - # Doing this in each loader, in case we are running from different processes, # avoiding to reload from same file (especially useful for boxed tests). # But deterministic path to avoid regenerating from the same interpreter @@ -468,14 +426,14 @@ def get_gen_path(self): def __repr__(self): return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - def exec_module(self, module): - # Custom implementation to declare an implicit namespace package - if (2, 7) <= sys.version_info < (3, 4) and self.implicit_ns_package: - module.__path__ = [self.path] # mandatory to trick importlib2 to think this is a package - import pkg_resources - pkg_resources.declare_namespace(module.__name__) - else: - super(ROSDefLoader, self).exec_module(module) + # def exec_module(self, module): + # # Custom implementation to declare an implicit namespace package + # if (2, 7) <= sys.version_info < (3, 4) and self.implicit_ns_package: + # module.__path__ = [self.path] # mandatory to trick importlib2 to think this is a package + # import pkg_resources + # pkg_resources.declare_namespace(module.__name__) + # else: + # super(ROSDefLoader, self).exec_module(module) else: raise ImportError("ros_loader : Unsupported python version") diff --git a/pyros_msgs/importer/tests/test_rosmsg_import.py b/pyros_msgs/importer/tests/test_rosmsg_import.py index 532a339..cad9565 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_import.py @@ -71,10 +71,12 @@ def print_importers(): class TestImplicitNamespace(unittest.TestCase): @classmethod def setUpClass(cls): - # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package - supported_loaders = rosmsg_finder._get_supported_ns_loaders() - ns_hook = rosmsg_finder.FileFinder2.path_hook(*supported_loaders) - sys.path_hooks.insert(1, ns_hook) + # This is required only for old python + if sys.version_info < (3, 4): + supported_loaders = rosmsg_finder._get_supported_ns_loaders() + ns_hook = rosmsg_finder.FileFinder2.path_hook(*supported_loaders) + sys.path_hooks.insert(1, ns_hook) + # python3 implicit namespaces should work out of the box. def test_import_relative_ns_subpkg(self): """Verify that package is importable relatively""" @@ -130,7 +132,7 @@ class TestImportMsg(unittest.TestCase): def setUpClass(cls): # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package supported_loaders = rosmsg_finder._get_supported_ros_loaders() - ros_hook = rosmsg_finder.DirectoryFinder.path_hook(*supported_loaders) + ros_hook = rosmsg_finder.ROSDirectoryFinder.path_hook(*supported_loaders) sys.path_hooks.insert(1, ros_hook) sys.path.append(cls.rosdeps_path) @@ -224,7 +226,7 @@ class TestImportSrv(unittest.TestCase): def setUpClass(cls): # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package supported_loaders = rosmsg_finder._get_supported_ros_loaders() - ros_hook = rosmsg_finder.DirectoryFinder.path_hook(*supported_loaders) + ros_hook = rosmsg_finder.ROSDirectoryFinder.path_hook(*supported_loaders) sys.path_hooks.insert(1, ros_hook) sys.path.append(cls.rosdeps_path) diff --git a/pyros_msgs/importer/tests/test_rosmsg_importlib.py b/pyros_msgs/importer/tests/test_rosmsg_importlib.py index bcf24c0..190a63f 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_importlib.py +++ b/pyros_msgs/importer/tests/test_rosmsg_importlib.py @@ -157,7 +157,7 @@ class TestImportLibAnotherMsg(unittest.TestCase): def setUpClass(cls): # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package supported_loaders = rosmsg_finder._get_supported_ros_loaders() - ros_hook = rosmsg_finder.DirectoryFinder.path_hook(*supported_loaders) + ros_hook = rosmsg_finder.ROSDirectoryFinder.path_hook(*supported_loaders) sys.path_hooks.insert(1, ros_hook) sys.path.append(cls.rosdeps_path) From c6b2654177f2929328e2e1aac8daf276ff3e2212 Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 27 Jun 2017 16:08:30 +0900 Subject: [PATCH 16/28] some cleanup --- .gitmodules | 6 --- pydeps/importlib2-fixed | 1 - pydeps/pytest-boxed | 1 - pyros_msgs/importer/importlib2 | 1 - pyros_msgs/importer/rosmsg_finder.py | 71 ++-------------------------- pyros_msgs/importer/rosmsg_loader.py | 32 ++----------- tox.ini | 2 +- 7 files changed, 7 insertions(+), 107 deletions(-) delete mode 160000 pydeps/importlib2-fixed delete mode 160000 pydeps/pytest-boxed delete mode 120000 pyros_msgs/importer/importlib2 diff --git a/.gitmodules b/.gitmodules index f4d3401..2351cfc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,12 +7,6 @@ [submodule "rosdeps/std_msgs"] path = rosdeps/std_msgs url = https://github.com/ros/std_msgs.git -[submodule "pydeps/pytest-boxed"] - path = pydeps/pytest-boxed - url = https://github.com/pytest-dev/pytest-boxed [submodule "rosdeps/ros_comm_msgs"] path = rosdeps/ros_comm_msgs url = https://github.com/ros/ros_comm_msgs.git -[submodule "pydeps/importlib2-fixed"] - path = pydeps/importlib2-fixed - url = https://github.com/pombredanne/ericsnowcurrently-importlib2.git diff --git a/pydeps/importlib2-fixed b/pydeps/importlib2-fixed deleted file mode 160000 index 3279cad..0000000 --- a/pydeps/importlib2-fixed +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3279cadc1d03421072f595a8ee8197919e906b1e diff --git a/pydeps/pytest-boxed b/pydeps/pytest-boxed deleted file mode 160000 index 9cf9bcc..0000000 --- a/pydeps/pytest-boxed +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9cf9bccf73ae616729c0f2b799c457d3b985e47f diff --git a/pyros_msgs/importer/importlib2 b/pyros_msgs/importer/importlib2 deleted file mode 120000 index fb69b6d..0000000 --- a/pyros_msgs/importer/importlib2 +++ /dev/null @@ -1 +0,0 @@ -../../pydeps/importlib2-fixed/importlib2 \ No newline at end of file diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py index 47818c8..77255d3 100644 --- a/pyros_msgs/importer/rosmsg_finder.py +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -31,73 +31,8 @@ import pkg_resources # useful to have empty directory imply namespace package (like for py3) from .rosmsg_loader import _ImportError, _verbose_message, FileLoader2 - import imp - # def spec_from_file_location(name, location=None, loader=None, - # submodule_search_locations=None): - # """Return a module spec based on a file location. - # - # To indicate that the module is a package, set - # submodule_search_locations to a list of directory paths. An - # empty list is sufficient, though its not otherwise useful to the - # import system. - # - # The loader must take a spec as its only __init__() arg. - # - # """ - # - # if location is None: - # # The caller may simply want a partially populated location- - # # oriented spec. So we set the location to a bogus value and - # # fill in as much as we can. - # location = '' - # if hasattr(loader, 'get_filename'): - # # ExecutionLoader - # try: - # location = loader.get_filename(name) - # except ImportError: - # pass - # - # # If the location is on the filesystem, but doesn't actually exist, - # # we could return None here, indicating that the location is not - # # valid. However, we don't have a good way of testing since an - # # indirect location (e.g. a zip file or URL) will look like a - # # non-existent file relative to the filesystem. - # - # spec = ModuleSpec(name, loader, origin=location) - # spec._set_fileattr = True - # - # # Pick a loader if one wasn't provided. - # if loader is None: - # for loader_class, suffixes in _get_supported_file_loaders(): - # if location.endswith(tuple(suffixes)): - # loader = loader_class(name, location) - # spec.loader = loader - # break - # else: - # return None - # - # # Set submodule_search_paths appropriately. - # if submodule_search_locations is None: - # # Check the loader. - # if hasattr(loader, 'is_package'): - # try: - # is_package = loader.is_package(name) - # except ImportError: - # pass - # else: - # if is_package: - # spec.submodule_search_locations = [] - # else: - # spec.submodule_search_locations = submodule_search_locations - # if spec.submodule_search_locations == []: - # if location: - # dirname = os.path.split(location)[0] - # spec.submodule_search_locations.append(dirname) - # - # return spec - class FileFinder2(object): def __init__(self, path, *loader_details): """Initialize with the path to search on and a variable number of @@ -235,7 +170,7 @@ def path_hook_for_ROSDirectoryFinder(path): import importlib.util as importlib_util - class DirectoryFinder(importlib_machinery.FileFinder): + class ROSDirectoryFinder(importlib_machinery.FileFinder): """Finder to interpret directories as modules, and files as classes""" def __init__(self, path, *ros_loader_details): @@ -263,7 +198,7 @@ def __init__(self, path, *ros_loader_details): ) def __repr__(self): - return 'DirectoryFinder({!r})'.format(self.path) + return 'ROSDirectoryFinder({!r})'.format(self.path) def find_spec(self, fullname, target=None): """ @@ -298,7 +233,7 @@ def find_spec(self, fullname, target=None): # since the plan is to eventually not have to rely on files at all TODO # Relying on FileFinder if we couldn't find any specific directory structure/content - # It will return a namespace spec if no file can be found (even with python2.7, thanks to importlib2) + # It will return a namespace spec if no file can be found # or will return a proper loader for already generated python files spec = spec or super(ROSDirectoryFinder, self).find_spec(fullname, target=target) # we return None if we couldn't find a spec before diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index d6708d3..5daf282 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -220,13 +220,6 @@ def get_gen_path(self): def __repr__(self): return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - # def get_filename(self, fullname): - # """Return the path to the source file.""" - # if os.path.isdir(self.path) and os.path.isfile(os.path.join(self.path, '__init__.py')): - # return os.path.join(self.path, '__init__.py') # python package - # else: - # return self.path # module or namespace package case - def __init__(self, fullname, path): self.logger = logging.getLogger(__name__) @@ -311,24 +304,17 @@ def __init__(self, fullname, path): # The file should have already been generated (by the loader for a msg package) # Note we do not want to rely on namespace packages here, since they are not standardized for python2, # and they can prevent some useful usecases. - + # TODO : This seems to be not used. confirm and cleanup # Hack to be able to "import generated classes" modname = fullname.rpartition('.')[2] filepath = os.path.join(self.outdir_pkg, loader_generated_subdir, '_' + modname + '.py') # the generated module # relying on usual source file loader since we have previously generated normal python code super(ROSDefLoader, self).__init__(fullname, filepath) - # - # def get_gen_path(self): - # """Returning the generated path matching the import""" - # return os.path.join(self.outdir_pkg, loader_generated_subdir) - # - # def __repr__(self): - # return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 (unsupported but might work ?) - import importlib.machinery as importlib_machinery + import importlib.machinery - class ROSDefLoader(importlib_machinery.SourceFileLoader): + class ROSDefLoader(importlib.machinery.SourceFileLoader): def __init__(self, fullname, path): self.logger = logging.getLogger(__name__) @@ -348,9 +334,6 @@ def __init__(self, fullname, path): indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != loader_origin_subdir and not p.endswith(loader_file_extension)] self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) - # : hack to be able to import a generated class (if requested) - # self.requested_class = None - if os.path.isdir(path): if path.endswith(loader_origin_subdir) and any([f.endswith(loader_file_extension) for f in os.listdir(path)]): # if we get a non empty 'msg' folder init_path = os.path.join(self.outdir_pkg, loader_generated_subdir, '__init__.py') @@ -426,15 +409,6 @@ def get_gen_path(self): def __repr__(self): return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - # def exec_module(self, module): - # # Custom implementation to declare an implicit namespace package - # if (2, 7) <= sys.version_info < (3, 4) and self.implicit_ns_package: - # module.__path__ = [self.path] # mandatory to trick importlib2 to think this is a package - # import pkg_resources - # pkg_resources.declare_namespace(module.__name__) - # else: - # super(ROSDefLoader, self).exec_module(module) - else: raise ImportError("ros_loader : Unsupported python version") diff --git a/tox.ini b/tox.ini index ac53a56..8e93923 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ commands= # we want to make sure python finds the installed package in tox env # and doesn't confuse with pyc generated during dev (which happens if we use self test feature here) - py.test --pyargs pyros_msgs/importer/tests {posargs} + py.test --pyargs pyros_msgs/importer/tests {posargs} --boxed py.test --pyargs pyros_msgs/typecheck/tests {posargs} py.test --pyargs pyros_msgs/opt_as_array/tests {posargs} py.test --pyargs pyros_msgs/opt_as_nested/tests {posargs} From 457e5763d744e736a7c4f2590037aa54f4bf88e6 Mon Sep 17 00:00:00 2001 From: alexv Date: Wed, 28 Jun 2017 10:59:02 +0900 Subject: [PATCH 17/28] extracting filefinder to separate pip package --- pyros_msgs/importer/_utils.py | 24 +++ pyros_msgs/importer/rosmsg_finder.py | 73 +------- pyros_msgs/importer/rosmsg_loader.py | 169 +----------------- pyros_msgs/importer/tests/_utils.py | 14 ++ .../importer/tests/nspkg/subpkg/__init__.py | 7 - .../importer/tests/nspkg/subpkg/submodule.py | 4 - .../importer/tests/test_rosmsg_import.py | 77 +------- .../importer/tests/test_rosmsg_importlib.py | 162 ++++------------- setup.py | 1 + 9 files changed, 79 insertions(+), 452 deletions(-) create mode 100644 pyros_msgs/importer/_utils.py create mode 100644 pyros_msgs/importer/tests/_utils.py delete mode 100644 pyros_msgs/importer/tests/nspkg/subpkg/__init__.py delete mode 100644 pyros_msgs/importer/tests/nspkg/subpkg/submodule.py diff --git a/pyros_msgs/importer/_utils.py b/pyros_msgs/importer/_utils.py new file mode 100644 index 0000000..866aa5a --- /dev/null +++ b/pyros_msgs/importer/_utils.py @@ -0,0 +1,24 @@ +from __future__ import absolute_import, print_function + +import sys + +def _verbose_message(message, *args, **kwargs): + """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" + verbosity = kwargs.pop('verbosity', 1) + if sys.flags.verbose >= verbosity: + if not message.startswith(('#', 'import ')): + message = '# ' + message + print(message.format(*args), file=sys.stderr) + +if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? + + try: + ImportError('msg', name='name', path='path') + except TypeError: + class _ImportError(ImportError): + def __init__(self, *args, **kwargs): + self.name = kwargs.pop('name', None) + self.path = kwargs.pop('path', None) + super(_ImportError, self).__init__(*args, **kwargs) + else: + _ImportError = ImportError diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py index 77255d3..b0cb665 100644 --- a/pyros_msgs/importer/rosmsg_finder.py +++ b/pyros_msgs/importer/rosmsg_finder.py @@ -29,75 +29,10 @@ # from .importlib2 import machinery as importlib_machinery # from .importlib2 import util as importlib_util import pkg_resources # useful to have empty directory imply namespace package (like for py3) + import filefinder2 + from ._utils import _ImportError, _verbose_message - from .rosmsg_loader import _ImportError, _verbose_message, FileLoader2 - import imp - - class FileFinder2(object): - def __init__(self, path, *loader_details): - """Initialize with the path to search on and a variable number of - 2-tuples containing the loader and the file suffixes the loader - recognizes.""" - loaders = [] - for loader, suffixes in loader_details: - loaders.extend((suffix, loader) for suffix in suffixes) - self._loaders = loaders - # Base (directory) path - self.path = path or '.' - # Note : we are not playing with cache here (too complex to get right and not worth it for obsolete python) - - def find_module(self, fullname, path=None): - """Try to find a loader for the specified module, or the namespace - package portions. Returns loader.""" - path = path or self.path - tail_module = fullname.rpartition('.')[2] - - base_path = os.path.join(path, tail_module) - for suffix, loader_class in self._loaders: - full_path = None # adjusting path for package or file - if os.path.isdir(base_path) and os.path.isfile(os.path.join(base_path, '__init__' + suffix)): - return loader_class(fullname, base_path) # __init__.py path will be computed by the loader when needed - elif os.path.isfile(base_path + suffix): - return loader_class(fullname, base_path + suffix) - else: - if os.path.isdir(base_path): - # If a namespace package, return the path if we don't - # find a module in the next section. - _verbose_message('possible namespace for {}'.format(base_path)) - return FileLoader2(fullname, base_path) - - return None - - @classmethod - def path_hook(cls, *loader_details): - """A class method which returns a closure to use on sys.path_hook - which will return an instance using the specified loaders and the path - called on the closure. - - If the path called on the closure is not a directory, ImportError is - raised. - - """ - def path_hook_for_FileFinder2(path): - """Path hook for FileFinder2.""" - if not os.path.isdir(path): - raise _ImportError('only directories are supported', path=path) - return cls(path, *loader_details) - - return path_hook_for_FileFinder2 - - def __repr__(self): - return 'FileFinder2({!r})'.format(self.path) - - def _get_supported_ns_loaders(): - """Returns a list of file-based module loaders. - Each item is a tuple (loader, suffixes). - """ - loader = FileLoader2, [suffix for suffix, mode, type in imp.get_suffixes()] - return [loader] - - - class ROSDirectoryFinder(FileFinder2): + class ROSDirectoryFinder(filefinder2.FileFinder2): """Finder to interpret directories as modules, and files as classes""" def __init__(self, path, *ros_loader_details): @@ -120,7 +55,7 @@ def __init__(self, path, *ros_loader_details): # We rely on FileFinder and python loader to deal with our generated code super(ROSDirectoryFinder, self).__init__( path, - *_get_supported_ns_loaders() + *filefinder2.get_supported_ns_loaders() ) def __repr__(self): diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py index 5daf282..dc85e84 100644 --- a/pyros_msgs/importer/rosmsg_loader.py +++ b/pyros_msgs/importer/rosmsg_loader.py @@ -24,174 +24,8 @@ # Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) import os import sys - -# This will take the ROS distro version if ros has been setup -import genpy.generator -import genpy.generate_initpy - import logging -def _verbose_message(message, *args, **kwargs): - """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - verbosity = kwargs.pop('verbosity', 1) - if sys.flags.verbose >= verbosity: - if not message.startswith(('#', 'import ')): - message = '# ' + message - print(message.format(*args), file=sys.stderr) - -if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? - - import io - import errno - import imp - - try: - ImportError('msg', name='name', path='path') - except TypeError: - class _ImportError(ImportError): - def __init__(self, *args, **kwargs): - self.name = kwargs.pop('name', None) - self.path = kwargs.pop('path', None) - super(_ImportError, self).__init__(*args, **kwargs) - else: - _ImportError = ImportError - - # Frame stripping magic ############################################### - - # def _call_with_frames_removed(f, *args, **kwds): - # """remove_importlib_frames in import.c will always remove sequences - # of importlib frames that end with a call to this function - # - # Use it instead of a normal call in places where including the importlib - # frames introduces unwanted noise into the traceback (e.g. when executing - # module code) - # """ - # return f(*args, **kwds) - # - # def decode_source(source_bytes): - # """Decode bytes representing source code and return the string. - # - # Universal newline support is used in the decoding. - # """ - # import tokenize # To avoid bootstrap issues. - # source_bytes_readline = io.BytesIO(source_bytes).readline - # encoding = tokenize.detect_encoding(source_bytes_readline) - # newline_decoder = io.IncrementalNewlineDecoder(None, True) - # return newline_decoder.decode(source_bytes.decode(encoding[0])) - - # inspired from importlib2 - class FileLoader2(object): - """Base file loader class which implements the loader protocol methods that - require file system usage. Also implements implicit namespace package PEP 420. - - CAREFUL: the finder/loader logic is DIFFERENT than for python3. - A namespace package, or a normal package, need to return a directory path. - get_filename() will append '__init__.py' if it exists and needs to be called when a module path is needed - """ - - def __init__(self, fullname, path): - """Cache the module name and the path to the file found by the - finder. - :param fullname the name of the module to load - :param path to the module or the package. - If it is a package (namespace or not) the path must point to a directory. - Otherwise the path to the python module is needed - """ - self.name = fullname - self.path = path - # Note for namespace package, we let pkg_resources manage the __path__ logic - - def __eq__(self, other): - return (self.__class__ == other.__class__ and - self.__dict__ == other.__dict__) - - def __hash__(self): - return hash(self.name) ^ hash(self.path) - - def get_source(self, fullname): - """Concrete implementation of InspectLoader.get_source.""" - path = self.get_filename(fullname) - try: - source_bytes = self.get_data(path) - except OSError as exc: - e = _ImportError('source not available through get_data()', - name=fullname) - e.__cause__ = exc - raise e - return source_bytes - # return decode_source(source_bytes) - - def load_module(self, name): - """Load a module from a file. - """ - # Implementation inspired from pytest.rewrite and importlib - - # If there is an existing module object named 'name' in - # sys.modules, the loader must use that existing module. (Otherwise, - # the reload() builtin will not work correctly.) - if name in sys.modules: - return sys.modules[name] - - code = self.get_code(name) - if code is None: - raise ImportError('cannot load module {!r} when get_code() ' - 'returns None'.format(name)) - # I wish I could just call imp.load_compiled here, but __file__ has to - # be set properly. In Python 3.2+, this all would be handled correctly - # by load_compiled. - mod = sys.modules.setdefault(name, imp.new_module(name)) - try: - # Set a few properties required by PEP 302 - mod.__file__ = self.get_filename(name) - mod.__loader__ = self - if self.is_package(name): - mod.__path__ = [self.path] - mod.__package__ = name # PEP 366 - else: - mod.__package__ = '.'.join(name.split('.')[:-1]) # PEP 366 - # if we want to skip compilation - useful for debugging - source = self.get_source(name) - exec (source, mod.__dict__) - #exec(code, mod.__dict__) - - except: - if name in sys.modules: - del sys.modules[name] - raise - return sys.modules[name] - - def get_filename(self, fullname): - """Return the path to the source file.""" - if os.path.isdir(self.path) and os.path.isfile(os.path.join(self.path, '__init__.py')): - return os.path.join(self.path, '__init__.py') # python package - else: - return self.path # module or namespace package case - - def get_code(self, fullname): - source = self.get_source(fullname) - print('compiling code for "%s"' % fullname) - return compile(source, self.get_filename(fullname), 'exec', dont_inherit=True) - - def is_package(self, fullname): - # TODO : test this (without ROS loader) - # in case of package we have to always have the directory as self.path - # we can always compute init path dynamically when needed. - return os.path.isdir(self.path) - - def get_data(self, path): - """Return the data from path as raw bytes. - Implements PEP 420 using pkg_resources - """ - try: - with io.FileIO(path, 'r') as file: - return file.read() - except IOError as ioe: - if ioe.errno == errno.EISDIR: - # implicit namespace package - return """import pkg_resources; pkg_resources.declare_namespace(__name__)""" - else: - raise - def RosLoader(rosdef_extension): """ @@ -210,8 +44,9 @@ def RosLoader(rosdef_extension): raise RuntimeError("RosLoader for a format {0} other than .msg or .srv is not supported".format(rosdef_extension)) if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? + import filefinder2 - class ROSDefLoader(FileLoader2): + class ROSDefLoader(filefinder2.FileLoader2): def get_gen_path(self): """Returning the generated path matching the import""" diff --git a/pyros_msgs/importer/tests/_utils.py b/pyros_msgs/importer/tests/_utils.py new file mode 100644 index 0000000..22c3667 --- /dev/null +++ b/pyros_msgs/importer/tests/_utils.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import, print_function + + +def print_importers(): + import sys + import pprint + + print('PATH:'), + pprint.pprint(sys.path) + print() + print('IMPORTERS:') + for name, cache_value in sys.path_importer_cache.items(): + name = name.replace(sys.prefix, '...') + print('%s: %r' % (name, cache_value)) diff --git a/pyros_msgs/importer/tests/nspkg/subpkg/__init__.py b/pyros_msgs/importer/tests/nspkg/subpkg/__init__.py deleted file mode 100644 index df5fa9d..0000000 --- a/pyros_msgs/importer/tests/nspkg/subpkg/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from __future__ import absolute_import - -from . import submodule - -# Just Dummy class for testing -class TestClassInSubPkg: - pass diff --git a/pyros_msgs/importer/tests/nspkg/subpkg/submodule.py b/pyros_msgs/importer/tests/nspkg/subpkg/submodule.py deleted file mode 100644 index 43b1032..0000000 --- a/pyros_msgs/importer/tests/nspkg/subpkg/submodule.py +++ /dev/null @@ -1,4 +0,0 @@ - -# Just Dummy class for testing -class TestClassInSubModule: - pass \ No newline at end of file diff --git a/pyros_msgs/importer/tests/test_rosmsg_import.py b/pyros_msgs/importer/tests/test_rosmsg_import.py index cad9565..7c7c888 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_import.py +++ b/pyros_msgs/importer/tests/test_rosmsg_import.py @@ -46,82 +46,7 @@ # https://pymotw.com/2/importlib/index.html -# HACK to fix spec from pytest -# @property -# def spec(): -# importlib.util.spec_from_file_location(__file__) - - -def print_importers(): - import sys - import pprint - - print('PATH:'), - pprint.pprint(sys.path) - print() - print('IMPORTERS:') - for name, cache_value in sys.path_importer_cache.items(): - name = name.replace(sys.prefix, '...') - print('%s: %r' % (name, cache_value)) - - -# We need to test implicit namespace packages PEP 420 (especially for python 2.7) -# Since we rely on it for ros import. -# But we can only teet relative package structure -class TestImplicitNamespace(unittest.TestCase): - @classmethod - def setUpClass(cls): - # This is required only for old python - if sys.version_info < (3, 4): - supported_loaders = rosmsg_finder._get_supported_ns_loaders() - ns_hook = rosmsg_finder.FileFinder2.path_hook(*supported_loaders) - sys.path_hooks.insert(1, ns_hook) - # python3 implicit namespaces should work out of the box. - - def test_import_relative_ns_subpkg(self): - """Verify that package is importable relatively""" - print_importers() - - from .nspkg import subpkg as test_pkg - - self.assertTrue(test_pkg is not None) - self.assertTrue(test_pkg.TestClassInSubPkg is not None) - self.assertTrue(callable(test_pkg.TestClassInSubPkg)) - - def test_import_relative_ns_subpkg_submodule(self): - """Verify that package is importable relatively""" - print_importers() - - from .nspkg.subpkg import submodule as test_mod - - self.assertTrue(test_mod is not None) - self.assertTrue(test_mod.TestClassInSubModule is not None) - self.assertTrue(callable(test_mod.TestClassInSubModule)) - - def test_import_class_from_relative_ns_subpkg(self): - """Verify that message class is importable relatively""" - print_importers() - - from .nspkg.subpkg import TestClassInSubPkg - - self.assertTrue(TestClassInSubPkg is not None) - self.assertTrue(callable(TestClassInSubPkg)) - - def test_import_class_from_relative_ns_subpkg_submodule(self): - """Verify that package is importable relatively""" - print_importers() - - from .nspkg.subpkg.submodule import TestClassInSubModule - - self.assertTrue(TestClassInSubModule is not None) - self.assertTrue(callable(TestClassInSubModule)) - - def test_import_relative_nonnspkg_raises(self): - """Verify that package is importable relatively""" - print_importers() - - with self.assertRaises(ImportError): - from .bad_nspkg import bad_subpkg +from ._utils import print_importers class TestImportMsg(unittest.TestCase): diff --git a/pyros_msgs/importer/tests/test_rosmsg_importlib.py b/pyros_msgs/importer/tests/test_rosmsg_importlib.py index 190a63f..87330c7 100644 --- a/pyros_msgs/importer/tests/test_rosmsg_importlib.py +++ b/pyros_msgs/importer/tests/test_rosmsg_importlib.py @@ -1,6 +1,6 @@ from __future__ import absolute_import, division, print_function """ -Testing executing rosmsg_generator directly (like setup.py would) +Testing dynamic import with importlib """ import os @@ -27,7 +27,7 @@ }, }) -# Since test frameworks (like pytest) play with the import machinery, we cannot use it here... +# Relying on basic unittest first, to be able to easily switch the test framework in case of import conflicts. import unittest # Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path @@ -41,113 +41,13 @@ # https://pymotw.com/3/importlib/index.html # https://pymotw.com/2/importlib/index.html -# if sys.version_info > (3, 4): -# import importlib.util -# # REF : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module -# # Useful to display details when debugging... -# def import_module(name, package=None): -# """An approximate implementation of import.""" -# absolute_name = importlib.util.resolve_name(name, package) -# try: -# return sys.modules[absolute_name] -# except KeyError: -# pass -# -# path = None -# if '.' in absolute_name: -# parent_name, _, child_name = absolute_name.rpartition('.') -# parent_module = import_module(parent_name) -# path = parent_module.spec.submodule_search_locations # this currently breaks (probably because of pytest custom importer ? ) -# for finder in sys.meta_path: -# spec = finder.find_spec(absolute_name, path) -# if spec is not None: -# break -# else: -# raise ImportError('No module named {absolute_name!r}'.format(**locals())) -# module = importlib.util.module_from_spec(spec) -# spec.loader.exec_module(module) -# sys.modules[absolute_name] = module -# if path is not None: -# setattr(parent_module, child_name, module) -# return module - - -# if sys.version_info > (3, 4): -# import importlib.util -# -# # Adapted from : https://docs.python.org/3.6/library/importlib.html#approximating-importlib-import-module -# # Useful to debug details... -# def import_module(name, package=None): -# # using find_spec to use our finder -# spec = importlib.util.find_spec(name, package) -# -# # path = None -# # if '.' in absolute_name: -# # parent_name, _, child_name = absolute_name.rpartition('.') -# # # recursive import call -# # parent_module = import_module(parent_name) -# # # getting the path instead of relying on spec (not managed by pytest it seems...) -# # path = [os.path.join(p, child_name) for p in parent_module.__path__ if os.path.exists(os.path.join(p, child_name))] -# -# # getting spec with importlib.util (instead of meta path finder to avoid uncompatible pytest finder) -# #spec = importlib.util.spec_from_file_location(absolute_name, path[0]) -# parent_module = None -# child_name = None -# if spec is None: -# # We didnt find anything, but this is expected on ros packages that haven't been generated yet -# if '.' in name: -# parent_name, _, child_name = name.rpartition('.') -# # recursive import call -# parent_module = import_module(parent_name) -# # we can check if there is a module in there.. -# path = [os.path.join(p, child_name) -# for p in parent_module.__path__._path -# if os.path.exists(os.path.join(p, child_name))] -# # we attempt to get the spec from the first found location -# while path and spec is None: -# spec = importlib.util.spec_from_file_location(name, path[0]) -# path[:] = path[1:] -# else: -# raise ImportError -# -# # checking again in case spec has been modified -# if spec is not None: -# if spec.name not in sys.modules: -# module = importlib.util.module_from_spec(spec) -# spec.loader.exec_module(module) -# sys.modules[name] = module -# if parent_module is not None and child_name is not None: -# setattr(parent_module, child_name, sys.modules[name]) -# return sys.modules[name] -# else: -# raise ImportError -# else: -# def import_module(name, package=None): -# return importlib.import_module(name=name, package=package) - - # # Note : we cannot assume anything about import implementation (different python version, different version of pytest) # => we need to test them all... # -# HACK to fix spec from pytest -# @property -# def spec(): -# importlib.util.spec_from_file_location(__file__) - - -def print_importers(): - import sys - import pprint - print('PATH:'), - pprint.pprint(sys.path) - print() - print('IMPORTERS:') - for name, cache_value in sys.path_importer_cache.items(): - name = name.replace(sys.prefix, '...') - print('%s: %r' % (name, cache_value)) +from ._utils import print_importers class TestImportLibAnotherMsg(unittest.TestCase): @@ -187,33 +87,33 @@ def test_importlib_import_absolute_class_raises(self): with self.assertRaises(ImportError): importlib.__import__('std_msgs.msg.Bool') - # BROKEN 3.4 - # @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - # def test_importlib_import_relative_pkg(self): - # # Verify that files exists and are importable - # test_msgs = importlib.__import__('.msg') - # - # self.assertTrue(test_msgs is not None) - # self.assertTrue(test_msgs.TestMsg is not None) - # self.assertTrue(callable(test_msgs.TestMsg)) - # self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name - # - # # use it ! - # self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) + # BROKEN 3.4 ? + @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + def test_importlib_import_relative_pkg(self): + # Verify that files exists and are importable + test_msgs = importlib.__import__('msg', globals=globals(), level=1) - # BROKEN 3.4 - # @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - # def test_importlib_import_relative_mod(self): - # # Verify that files exists and are importable - # msg = importlib.__import__('.msg.TestMsg') - # TestMsg = msg.TestMsg - # - # self.assertTrue(TestMsg is not None) - # self.assertTrue(callable(TestMsg)) - # self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name - # - # # use it ! - # self.assertTrue(TestMsg(test_bool=True, test_string='Test').test_bool) + self.assertTrue(test_msgs is not None) + self.assertTrue(test_msgs.TestMsg is not None) + self.assertTrue(callable(test_msgs.TestMsg)) + self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name + + # use it ! + self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) + + # BROKEN 3.4 ? + @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") + def test_importlib_import_relative_mod(self): + # Verify that files exists and are importable + msg = importlib.__import__('msg.TestMsg', globals=globals(), level=1) + TestMsg = msg.TestMsg + + self.assertTrue(TestMsg is not None) + self.assertTrue(callable(TestMsg)) + self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name + + # use it ! + self.assertTrue(TestMsg(test_bool=True, test_string='Test').test_bool) @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), reason="importlib does not have attribute find_loader or load_module") @@ -356,6 +256,8 @@ def test_importlib_importmodule_absolute_class_raises(self): @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") def test_importlib_importmodule_relative_msg(self): + + assert __package__ # Verify that files exists and are dynamically importable test_msgs = importlib.import_module('.msg', package=__package__) @@ -378,6 +280,8 @@ def test_importlib_importmodule_relative_msg(self): @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") def test_importlib_importmodule_relative_class_raises(self): + + assert __package__ with self.assertRaises(ImportError): importlib.import_module('.msg.TestMsg', package=__package__) diff --git a/setup.py b/setup.py index e64d9b3..53439a5 100644 --- a/setup.py +++ b/setup.py @@ -249,6 +249,7 @@ def run(self): # 'pyros_setup>=0.2.1', # needed to grab ros environment even if distro setup.sh not sourced # 'pyros_utils', # this must be satisfied by the ROS package system... # 'importlib2>=3.4;python_version<"3.4"', # NOT working we use a patched version of it, through a symlink (to make linux deb release possible) + 'filefinder2; python_version<"3.4"', # we rely on this for PEP420 on python 2.7 'pyyaml>=3.10', # genpy relies on this... 'pytest>=2.8.0', # as per hypothesis requirement (careful with 2.5.1 on trusty) 'pytest-xdist', # for --boxed (careful with the version it will be moved out of xdist) From 79a01f0e543cfdf86b6b6cd04f2f176d72eb8b91 Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 14 Jul 2017 18:07:17 +0900 Subject: [PATCH 18/28] moving importer out to rosimport, and fixing tests. --- dev-requirements.txt | 8 +- pyros_msgs/importer/__init__.py | 39 --- pyros_msgs/importer/_utils.py | 24 -- pyros_msgs/importer/rosmsg_finder.py | 262 --------------- pyros_msgs/importer/rosmsg_generator.py | 306 ----------------- pyros_msgs/importer/rosmsg_loader.py | 253 -------------- pyros_msgs/importer/rosmsg_metafinder.py | 192 ----------- pyros_msgs/importer/tests/__init__.py | 23 -- pyros_msgs/importer/tests/_utils.py | 14 - pyros_msgs/importer/tests/msg/TestMsg.msg | 3 - pyros_msgs/importer/tests/srv/TestSrv.srv | 4 - .../tests/test_rosmsg_generator_exec.py | 71 ---- .../importer/tests/test_rosmsg_import.py | 280 ---------------- .../importer/tests/test_rosmsg_importlib.py | 314 ------------------ .../opt_as_array/tests/test_opt_bool.py | 14 +- .../opt_as_array/tests/test_opt_duration.py | 20 +- .../opt_as_array/tests/test_opt_int16.py | 25 +- .../opt_as_array/tests/test_opt_int32.py | 25 +- .../opt_as_array/tests/test_opt_int64.py | 24 +- .../opt_as_array/tests/test_opt_int8.py | 26 +- .../opt_as_array/tests/test_opt_std_empty.py | 15 +- .../opt_as_array/tests/test_opt_string.py | 24 +- .../opt_as_array/tests/test_opt_time.py | 20 +- .../opt_as_array/tests/test_opt_uint16.py | 26 +- .../opt_as_array/tests/test_opt_uint32.py | 25 +- .../opt_as_array/tests/test_opt_uint64.py | 26 +- .../opt_as_array/tests/test_opt_uint8.py | 26 +- .../opt_as_nested/tests/msg_generate.py | 118 +++---- .../opt_as_nested/tests/test_opt_bool.py | 22 +- .../opt_as_nested/tests/test_opt_duration.py | 22 +- .../opt_as_nested/tests/test_opt_int16.py | 16 +- .../opt_as_nested/tests/test_opt_int32.py | 17 +- .../opt_as_nested/tests/test_opt_int64.py | 17 +- .../opt_as_nested/tests/test_opt_int8.py | 22 +- .../opt_as_nested/tests/test_opt_std_empty.py | 22 +- .../opt_as_nested/tests/test_opt_string.py | 28 +- .../opt_as_nested/tests/test_opt_time.py | 23 +- .../opt_as_nested/tests/test_opt_uint16.py | 17 +- .../opt_as_nested/tests/test_opt_uint32.py | 16 +- .../opt_as_nested/tests/test_opt_uint64.py | 17 +- .../opt_as_nested/tests/test_opt_uint8.py | 16 +- pyros_msgs/typecheck/tests/msg_generate.py | 35 +- .../tests/test_basic_ros_serialization.py | 24 +- .../typecheck/tests/test_ros_mappings.py | 20 +- .../tests/test_typechecker_arrays.py | 8 +- .../typecheck/tests/test_typechecker_basic.py | 16 +- 46 files changed, 376 insertions(+), 2189 deletions(-) delete mode 100644 pyros_msgs/importer/__init__.py delete mode 100644 pyros_msgs/importer/_utils.py delete mode 100644 pyros_msgs/importer/rosmsg_finder.py delete mode 100644 pyros_msgs/importer/rosmsg_generator.py delete mode 100644 pyros_msgs/importer/rosmsg_loader.py delete mode 100644 pyros_msgs/importer/rosmsg_metafinder.py delete mode 100644 pyros_msgs/importer/tests/__init__.py delete mode 100644 pyros_msgs/importer/tests/_utils.py delete mode 100644 pyros_msgs/importer/tests/msg/TestMsg.msg delete mode 100644 pyros_msgs/importer/tests/srv/TestSrv.srv delete mode 100644 pyros_msgs/importer/tests/test_rosmsg_generator_exec.py delete mode 100644 pyros_msgs/importer/tests/test_rosmsg_import.py delete mode 100644 pyros_msgs/importer/tests/test_rosmsg_importlib.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 4ec6317..ebc207a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1 +1,7 @@ -#pathlib2 \ No newline at end of file +#pathlib2 +gitchangelog +twine + +# source access to latest filefinder from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport diff --git a/pyros_msgs/importer/__init__.py b/pyros_msgs/importer/__init__.py deleted file mode 100644 index 90a7037..0000000 --- a/pyros_msgs/importer/__init__.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import absolute_import - - -# This is useful only if we need relative imports. Ref : http://stackoverflow.com/a/28154841/4006172 -# declaring __package__ if needed (this module is run individually) -if __package__ is None and not __name__.startswith('pyros_msgs.importer.'): - import os - import sys - # from pathlib2 import Path - # top = Path(__file__).resolve().parents[2] - # Or - from os.path import abspath, dirname - top = abspath(__file__) - for _ in range(4): - top = dirname(top) - - if sys.path[0] == os.path.dirname(__file__): - sys.path[0] = str( - top) # we replace first path in list (current module dir path) by the path of the package. - # this avoid unintentional relative import (even without point notation). - else: # not sure in which case this could happen, but just in case we don't want to break stuff - sys.path.append(str(top)) - - if __name__ == '__main__': - __name__ = '__init__' - - __package__ = 'pyros_msgs.importer' - # Note we do NOT want to import everything in pyros_msgs in this case - - -from .rosmsg_generator import ( - MsgDependencyNotFound, - generate_msgsrv_nspkg -) - -__all__ = [ - 'MsgDependencyNotFound', - 'generate_msgsrv_nspkg', -] diff --git a/pyros_msgs/importer/_utils.py b/pyros_msgs/importer/_utils.py deleted file mode 100644 index 866aa5a..0000000 --- a/pyros_msgs/importer/_utils.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import absolute_import, print_function - -import sys - -def _verbose_message(message, *args, **kwargs): - """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" - verbosity = kwargs.pop('verbosity', 1) - if sys.flags.verbose >= verbosity: - if not message.startswith(('#', 'import ')): - message = '# ' + message - print(message.format(*args), file=sys.stderr) - -if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? - - try: - ImportError('msg', name='name', path='path') - except TypeError: - class _ImportError(ImportError): - def __init__(self, *args, **kwargs): - self.name = kwargs.pop('name', None) - self.path = kwargs.pop('path', None) - super(_ImportError, self).__init__(*args, **kwargs) - else: - _ImportError = ImportError diff --git a/pyros_msgs/importer/rosmsg_finder.py b/pyros_msgs/importer/rosmsg_finder.py deleted file mode 100644 index b0cb665..0000000 --- a/pyros_msgs/importer/rosmsg_finder.py +++ /dev/null @@ -1,262 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import contextlib -import importlib -import site - -from pyros_msgs.importer import rosmsg_loader - -""" -A module to setup custom importer for .msg and .srv files -Upon import, it will first find the .msg file, then generate the python module for it, then load it. - -TODO... -""" - -# We need to be extra careful with python versions -# Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path -# Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) -import os -import sys - -import logging - -from .rosmsg_loader import ROSMsgLoader, ROSSrvLoader - -if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? - # from .importlib2 import machinery as importlib_machinery - # from .importlib2 import util as importlib_util - import pkg_resources # useful to have empty directory imply namespace package (like for py3) - import filefinder2 - from ._utils import _ImportError, _verbose_message - - class ROSDirectoryFinder(filefinder2.FileFinder2): - """Finder to interpret directories as modules, and files as classes""" - - def __init__(self, path, *ros_loader_details): - """ - Finder to get directories containing ROS message and service files. - It need to be inserted in sys.path_hooks before FileFinder, since these are Directories but not containing __init__ as per python hardcoded convention. - - Note: There is a matching issue between msg/ folder and msg/My.msg on one side, and package, module, class concepts on the other. - Since a module is not callable, but we need to call My(data) to build a message class (ROS convention), we match the msg/ folder to a module (and not a package) - And to keep matching ROS conventions, a directory without __init__ or any message/service file, will become a namespace (sub)package. - - :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) - """ - - ros_loaders = [] - for loader, suffixes in ros_loader_details: - ros_loaders.extend((suffix, loader) for suffix in suffixes) - self._ros_loaders = ros_loaders - - # We rely on FileFinder and python loader to deal with our generated code - super(ROSDirectoryFinder, self).__init__( - path, - *filefinder2.get_supported_ns_loaders() - ) - - def __repr__(self): - return 'ROSDirectoryFinder({!r})'.format(self.path) - - def find_module(self, fullname, path=None): - path = path or self.path - tail_module = fullname.rpartition('.')[2] - loader = None - - base_path = os.path.join(path, tail_module) - # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. - if os.path.isdir(base_path): - loader_class = None - rosdir = None - # Figuring out if we should care about this directory at all - for root, dirs, files in os.walk(base_path): - for suffix, loader_cls in self._ros_loaders: - if any(f.endswith(suffix) for f in files): - loader_class = loader_cls - rosdir = root - if loader_class and rosdir and rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module - # Generate something ! - # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location - loader = loader_class(fullname, base_path) # loader.get_gen_path()]) - # We DO NOT WANT TO add the generated dir in sys.path to use a python loader - # since the plan is to eventually not have to rely on files at all TODO - - # if we couldn't build a loader before we forward the call to our parent FileFinder2 (useful for implicit namespace packages) - loader = loader or super(ROSDirectoryFinder, self).find_module(fullname, path) - # If we couldnt find any loader before, we return None - return loader - - @classmethod - def path_hook(cls, *loader_details): - # Same as FileFinder2 - def path_hook_for_ROSDirectoryFinder(path): - """Path hook for ROSDirectoryFinder.""" - if not os.path.isdir(path): - raise _ImportError('only directories are supported', path=path) - return cls(path, *loader_details) - - return path_hook_for_ROSDirectoryFinder - -elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 (maybe we could, if it was worth it...) - import importlib.machinery as importlib_machinery - import importlib.util as importlib_util - - - class ROSDirectoryFinder(importlib_machinery.FileFinder): - """Finder to interpret directories as modules, and files as classes""" - - def __init__(self, path, *ros_loader_details): - """ - Finder to get directories containing ROS message and service files. - It need to be inserted in sys.path_hooks before FileFinder, since these are Directories but not containing __init__ as per python hardcoded convention. - - Note: There is a matching issue between msg/ folder and msg/My.msg on one side, and package, module, class concepts on the other. - Since a module is not callable, but we need to call My(data) to build a message class (ROS convention), we match the msg/ folder to a module (and not a package) - And to keep matching ROS conventions, a directory without __init__ or any message/service file, will become a namespace (sub)package. - - :param path_entry: the msg or srv directory path (no finder should have been instantiated yet) - """ - - ros_loaders = [] - for loader, suffixes in ros_loader_details: - ros_loaders.extend((suffix, loader) for suffix in suffixes) - self._ros_loaders = ros_loaders - - # We rely on FileFinder and python loader to deal with our generated code - super(ROSDirectoryFinder, self).__init__( - path, - (importlib_machinery.SourceFileLoader, ['.py']), - (importlib_machinery.SourcelessFileLoader, ['.pyc']), - ) - - def __repr__(self): - return 'ROSDirectoryFinder({!r})'.format(self.path) - - def find_spec(self, fullname, target=None): - """ - Try to find a spec for the specified module. - Returns the matching spec, or None if not found. - :param fullname: the name of the package we are trying to import - :param target: what we plan to do with it - :return: - """ - - tail_module = fullname.rpartition('.')[2] - spec = None - base_path = os.path.join(self.path, tail_module) - - # special code here since FileFinder expect a "__init__" that we don't need for msg or srv. - if os.path.isdir(base_path): - loader_class = None - rosdir = None - # Figuring out if we should care about this directory at all - for root, dirs, files in os.walk(base_path): - for suffix, loader_cls in self._ros_loaders: - if any(f.endswith(suffix) for f in files): - loader_class = loader_cls - rosdir = root - if loader_class and rosdir: - if rosdir == base_path: # we found a message/service file in the hierarchy, that belong to our module - # Generate something ! - loader = loader_class(fullname, base_path) - # we are looking for submodules either in generated location (to be able to load generated python files) or in original msg location - spec = importlib_util.spec_from_file_location(fullname, base_path, loader=loader, submodule_search_locations=[base_path, loader.get_gen_path()]) - # We DO NOT WANT TO add the generated dir in sys.path to use a python loader - # since the plan is to eventually not have to rely on files at all TODO - - # Relying on FileFinder if we couldn't find any specific directory structure/content - # It will return a namespace spec if no file can be found - # or will return a proper loader for already generated python files - spec = spec or super(ROSDirectoryFinder, self).find_spec(fullname, target=target) - # we return None if we couldn't find a spec before - return spec - - -else: - raise ImportError("ros_loader : Unsupported python version") - - -MSG_SUFFIXES = ['.msg'] -SRV_SUFFIXES = ['.srv'] - -def _get_supported_ros_loaders(): - """Returns a list of file-based module loaders. - Each item is a tuple (loader, suffixes). - """ - msg = ROSMsgLoader, MSG_SUFFIXES - srv = ROSSrvLoader, SRV_SUFFIXES - return [msg, srv] - - -def _install(): - """Install the path-based import components.""" - supported_loaders = _get_supported_ros_loaders() - sys.path_hooks.extend([ROSDirectoryFinder.path_hook(*supported_loaders)]) - # TODO : sys.meta_path.append(DistroFinder) - - - -# Useless ? -#_ros_finder_instance_obsolete_python = ROSImportFinder - -ros_distro_finder = None - -# TODO : metafinder -def activate(rosdistro_path=None, *workspaces): - global ros_distro_finder - if rosdistro_path is None: # autodetect most recent installed distro - if os.path.exists('/opt/ros/lunar'): - rosdistro_path = '/opt/ros/lunar' - elif os.path.exists('/opt/ros/kinetic'): - rosdistro_path = '/opt/ros/kinetic' - elif os.path.exists('/opt/ros/jade'): - rosdistro_path = '/opt/ros/jade' - elif os.path.exists('/opt/ros/indigo'): - rosdistro_path = '/opt/ros/indigo' - else: - raise ImportError( - "No ROS distro detected on this system. Please specify the path when calling ROSImportMetaFinder()") - - ros_distro_finder = ROSImportMetaFinder(rosdistro_path, *workspaces) - sys.meta_path.append(ros_distro_finder) - - #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? - - # We need to be before FileFinder to be able to find our (non .py[c]) files - # inside, maybe already imported, python packages... - sys.path_hooks.insert(1, ROSImportFinder) - - # else: # older (trusty) version - # sys.path_hooks.append(_ros_finder_instance_obsolete_python) - - for hook in sys.path_hooks: - print('Path hook: {}'.format(hook)) - - # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths - if paths: - sys.path.append(*paths) - - -def deactivate(*paths): - """ CAREFUL : even if we remove our path_hooks, the created finder are still cached in sys.path_importer_cache.""" - #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? - sys.path_hooks.remove(ROSImportFinder) - # else: # older (trusty) version - # sys.path_hooks.remove(_ros_finder_instance_obsolete_python) - if paths: - sys.path.remove(*paths) - - sys.meta_path.remove(ros_distro_finder) - - -@contextlib.contextmanager -def ROSImportContext(*paths): - activate(*paths) - yield - deactivate(*paths) - - -# TODO : a meta finder could find a full ROS distro... diff --git a/pyros_msgs/importer/rosmsg_generator.py b/pyros_msgs/importer/rosmsg_generator.py deleted file mode 100644 index 4ed9361..0000000 --- a/pyros_msgs/importer/rosmsg_generator.py +++ /dev/null @@ -1,306 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import os -import sys -import tempfile -import traceback -import importlib - -import time - -import pkg_resources - -""" -Module that can be used standalone, or as part of the pyros_msgs.importer package - -It provides a set of functions to generate your ros messages, even when ROS is not installed on your system. -You might however need to have dependent messages definitions reachable by rospack somewhere. - -Note : Generated modules/packages can only be imported once. So it is important to provide an API that : -- makes it easy to generate the whole module/package at once, since this is our priority -- makes it easy to optionally import the whole generated module/package -- still allows to generate only one module / a part of the whole package, caveats apply / warning added. -""" - -try: - # Using genpy and genmsg directly if ROS has been setup (while using from ROS pkg) - import genmsg as genmsg - import genmsg.command_line as genmsg_command_line - import genpy.generator as genpy_generator - import genpy.generate_initpy as genpy_generate_initpy - -except ImportError: - - # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) - - import site - ros_site_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'ros-site') - print("Adding site directory {ros_site_dir} to access genpy and genmsg.".format(**locals())) - site.addsitedir(ros_site_dir) - - import genmsg as genmsg - import genmsg.command_line as genmsg_command_line - import genpy.generator as genpy_generator - import genpy.generate_initpy as genpy_generate_initpy - - # Note we do not want to use pyros_setup here. - # We do not want to do a full ROS setup, only import specific packages. - # If needed it should have been done before (loading a parent package). - # this handle the case where we want to be independent of any underlying ROS system. - - -class MsgDependencyNotFound(Exception): - pass - - -class PkgAlreadyExists(Exception): - pass - - -def genros_py(rosfiles, generator, package, outdir, includepath=None): - includepath = includepath or [] - - if not os.path.exists(outdir): - # This script can be run multiple times in parallel. We - # don't mind if the makedirs call fails because somebody - # else snuck in and created the directory before us. - try: - os.makedirs(outdir) - except OSError as e: - if not os.path.exists(outdir): - raise - # TODO : maybe we dont need this, and that translation should be handled before ? - search_path = genmsg.command_line.includepath_to_dict(includepath) - generator.generate_messages(package, rosfiles, outdir, search_path) - - -def genmsg_py(msg_files, package, outdir_pkg, includepath=None, initpy=True): - """ - Generates message modules for a package, in that package directory, in a subpackage called 'msg', following ROS conventions - :param msg_files: the .msg files to use as input for generating the python message classes - :param package: the package for which we want to generate these messages - :param outdir_pkg: the directory of the package, where to put these messages. It should finish with path - :param includepath: optionally the list of path to include, in order to retrieve message dependencies - :param initpy: whether create an __init__.py for this package - :return: the list of files generated - """ - includepath = includepath or [] - outdir = os.path.join(outdir_pkg, 'msg') - try: - genros_py(rosfiles=[f for f in msg_files if f.endswith('.msg')], - generator=genpy_generator.MsgGenerator(), - package=package, - outdir=outdir, - includepath=includepath, - ) - # because the OS interface might not be synchronous.... - while not os.path.exists(outdir): - time.sleep(.1) - - except genmsg.InvalidMsgSpec as e: - print("ERROR: ", e, file=sys.stderr) - raise - except genmsg.MsgGenerationException as e: - print("ERROR: ", e, file=sys.stderr) - raise - except Exception as e: - traceback.print_exc() - print("ERROR: ", e) - raise - - genset = set() - - # optionally we can generate __init__.py - if initpy: - init_path = os.path.join(outdir, '__init__.py') - if os.path.exists(init_path): - raise PkgAlreadyExists("Keeping {init_path}, generation skipped.") - else: - genpy_generate_initpy.write_modules(outdir) - genset.add(init_path) - else: # we list all files, only if init.py was not created (and user has to import one by one) - for f in msg_files: - f, _ = os.path.splitext(f) # removing extension - os.path.relpath(outdir) - genset.add(os.path.join(outdir, '_' + os.path.basename(f) + '.py')) - - return genset - - -def gensrv_py(srv_files, package, outdir_pkg, includepath=None, initpy=True): - """ - Generates service modules for a package, in that package directory, in a subpackage called 'srv', following ROS conventions - :param srv_files: the .srv files to use as input for generating the python service classes - :param package: the package for which we want to generate these services - :param outdir_pkg: the directory of the package, where to put these services. It should finish with path - :param includepath: optionally the list of path to include, in order to retrieve message dependencies - :param initpy: whether create an __init__.py for this package - :return: the list of files generated - """ - includepath = includepath or [] - outdir = os.path.join(outdir_pkg, 'srv') - try: - genros_py(rosfiles=[f for f in srv_files if f.endswith('.srv')], - generator=genpy_generator.SrvGenerator(), - package=package, - outdir=outdir, - includepath=includepath, - ) - # because the OS interface might not be synchronous.... - while not os.path.exists(outdir): - time.sleep(.1) - - except genmsg.InvalidMsgSpec as e: - print("ERROR: ", e, file=sys.stderr) - raise - except genmsg.MsgGenerationException as e: - print("ERROR: ", e, file=sys.stderr) - raise - except Exception as e: - traceback.print_exc() - print("ERROR: ", e) - raise - - genset = set() - - # optionally we can generate __init__.py - if initpy: - init_path = os.path.join(outdir, '__init__.py') - if os.path.exists(init_path): - raise PkgAlreadyExists("Keeping {init_path}, generation skipped.") - else: - genpy_generate_initpy.write_modules(outdir) - genset.add(init_path) - else: # we list all files, only if init.py was not created (and user has to import one by one) - for f in srv_files: - f, _ = os.path.splitext(f) # removing extension - os.path.relpath(outdir) - genset.add(os.path.join(outdir, '_' + os.path.basename(f) + '.py')) - - return genset - - -def genmsgsrv_py(msgsrv_files, package, outdir_pkg, includepath=None, ns_pkg=True): - """""" - includepath = includepath or [] - - # checking if we have files with unknown extension to except early - for f in msgsrv_files: - if not f.endswith(('.msg', '.srv')): - print("WARNING: {f} doesnt have the proper .msg or .srv extension. It has been Ignored.".format(**locals()), file=sys.stderr) - - genset = set() - generated_msg = genmsg_py(msg_files=[f for f in msgsrv_files if f.endswith('.msg')], - package=package, - outdir_pkg=outdir_pkg, - includepath=includepath, - initpy=True) # we always create an __init__.py when called from here. - generated_srv = gensrv_py(srv_files=[f for f in msgsrv_files if f.endswith('.srv')], - package=package, - outdir_pkg=outdir_pkg, - includepath=includepath, - initpy=True) # we always create an __init__.py when called from here. - - if ns_pkg: - # The namespace package creation is only here to allow mixing different path for the same package - # so that we do not have to generate messages and services in the package path (that might not be writeable) - # Note : the *first* package imported need to declare the namespace package for this to work. - # Ref : https://packaging.python.org/namespace_packages/ - nspkg_init_path = os.path.join(outdir_pkg, '__init__.py') - if os.path.exists(nspkg_init_path): - raise PkgAlreadyExists("Keeping {nspkg_init_path}, generation skipped.") - else: - with open(nspkg_init_path, "w") as nspkg_init: - nspkg_init.writelines([ - "from __future__ import absolute_import, division, print_function\n", - "# this is an autogenerated file for dynamic ROS message creation\n", - "import pkg_resources\n", - "pkg_resources.declare_namespace(__name__)\n", - "" - ]) - - # because the OS interface might not be synchronous.... - while not os.path.exists(nspkg_init_path): - time.sleep(.1) - - # Whether or not we create the namespace package, we have to return back both msg and srv subpackages, - # since they need to be imported explicitely - genset = genset.union(generated_msg) - genset = genset.union(generated_srv) - - return genset - - -def generate_msgsrv_nspkg(msgsrvfiles, package=None, dependencies=None, include_path=None, outdir_pkg=None, ns_pkg=True): - # TODO : since we return a full package, we should probably pass a dir, not the files one by one... - # by default we generate for this package (does it make sense ?) - # Careful it might still be None - package = package or 'gen_msgs' - - # by default we have no dependencies - dependencies = dependencies or [] - - if not outdir_pkg or not outdir_pkg.startswith(os.sep): - # if path is not absolute, we create a temporary directory to hold our generated package - gendir = tempfile.mkdtemp('pyros_gen_site') - outdir_pkg = os.path.join(gendir, outdir_pkg if outdir_pkg else package) - - include_path = include_path or [] - - # we might need to resolve some dependencies - unresolved_dependencies = [d for d in dependencies if d not in [p.split(':')[0] for p in include_path]] - - if unresolved_dependencies: - try: - # In that case we have no choice but to rely on ros packages (on the system) => ROS has to be setup. - import rospkg - - # get an instance of RosPack with the default search paths - rospack = rospkg.RosPack() - for d in unresolved_dependencies: - try: - # get the file path for a dependency - dpath = rospack.get_path(d) - # we populate include_path - include_path.append('{d}:{dpath}/msg'.format(**locals())) # AFAIK only msg can be dependent msg types - except rospkg.ResourceNotFound as rnf: - raise MsgDependencyNotFound(rnf.message) - except ImportError: - print("Attempt to import rospkg failed before attempting to resolve dependencies {0}".format(unresolved_dependencies)) - - gen_files = genmsgsrv_py(msgsrv_files=msgsrvfiles, package=package, outdir_pkg=outdir_pkg, includepath=include_path, ns_pkg=ns_pkg) - - # computing module names that will be importable after outdir_pkg has been added as sitedir - gen_msgs = None - gen_srvs = None - # Not ideal, but it will do until we implement a custom importer - for f in gen_files: - test_gen_msgs_parent = None - f = f[:-len('.py')] if f.endswith('.py') else f - f = f[:-len(os.sep + '__init__')] if f.endswith(os.sep + '__init__') else f - - if f.endswith('msg'): - gen_msgs = package + '.msg' - - if f.endswith('srv'): - gen_srvs = package + '.srv' - - # we need to get one level up to get the sitedir (containing the generated namespace package) - return os.path.dirname(outdir_pkg), gen_msgs, gen_srvs - # TODO : return something that can be imported later... with custom importer or following importlib API... - - -# This API is useful to import after a generation has been done with details. -# TODO : implement custom importer to do this as properly as possible -def import_msgsrv(sitedir, gen_msgs = None, gen_srvs = None): - import site - site.addsitedir(sitedir) # we add our output dir as a site (to be able to import from it as usual) - # because we modify sys.path, we also need to handle namespace packages - pkg_resources.fixup_namespace_packages(sitedir) - - msgs_mod = importlib.import_module(gen_msgs) - srvs_mod = importlib.import_module(gen_srvs) - - return msgs_mod, srvs_mod - # TODO : doublecheck and fix that API to return the same thing as importlib.import_module returns, for consistency,... - diff --git a/pyros_msgs/importer/rosmsg_loader.py b/pyros_msgs/importer/rosmsg_loader.py deleted file mode 100644 index dc85e84..0000000 --- a/pyros_msgs/importer/rosmsg_loader.py +++ /dev/null @@ -1,253 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import contextlib -import importlib -import site -import tempfile - -import shutil - - -from pyros_msgs.importer import rosmsg_generator - -""" -A module to setup custom importer for .msg and .srv files -Upon import, it will first find the .msg file, then generate the python module for it, then load it. - -TODO... -""" - -# We need to be extra careful with python versions -# Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path -# Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) -import os -import sys -import logging - - -def RosLoader(rosdef_extension): - """ - Function generating ROS loaders. - This is used to keep .msg and .srv loaders very similar - """ - if rosdef_extension == '.msg': - loader_origin_subdir = 'msg' - loader_file_extension = rosdef_extension - loader_generated_subdir = 'msg' - elif rosdef_extension == '.srv': - loader_origin_subdir = 'srv' - loader_file_extension = rosdef_extension - loader_generated_subdir = 'srv' - else: - raise RuntimeError("RosLoader for a format {0} other than .msg or .srv is not supported".format(rosdef_extension)) - - if (2, 7) <= sys.version_info < (3, 4): # valid until which py3 version ? - import filefinder2 - - class ROSDefLoader(filefinder2.FileLoader2): - - def get_gen_path(self): - """Returning the generated path matching the import""" - return os.path.join(self.outdir_pkg, loader_generated_subdir) - - def __repr__(self): - return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - - def __init__(self, fullname, path): - - self.logger = logging.getLogger(__name__) - - # Doing this in each loader, in case we are running from different processes, - # avoiding to reload from same file (especially useful for boxed tests). - # But deterministic path to avoid regenerating from the same interpreter - self.rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) - if os.path.exists(self.rosimport_path): - shutil.rmtree(self.rosimport_path) - os.makedirs(self.rosimport_path) - - self.rospackage = fullname.partition('.')[0] - # We should reproduce package structure in generated file structure - dirlist = path.split(os.sep) - pkgidx = dirlist[::-1].index(self.rospackage) - indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != loader_origin_subdir and not p.endswith(loader_file_extension)] - self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) - - # : hack to be able to import a generated class (if requested) - # self.requested_class = None - - if os.path.isdir(path): - if path.endswith(loader_origin_subdir) and any([f.endswith(loader_file_extension) for f in os.listdir(path)]): # if we get a non empty 'msg' folder - init_path = os.path.join(self.outdir_pkg, loader_generated_subdir, '__init__.py') - if not os.path.exists(init_path): - # TODO : we need to determine that from the loader - # as a minimum we need to add current package - self.includepath = [self.rospackage + ':' + path] # TODO :maybe keep a list of all messages we have imported (sys.rosmessages) - - # TODO : unify this after reviewing rosmsg_generator API - if loader_file_extension == '.msg': - # TODO : dynamic in memory generation (we do not need the file ultimately...) - self.gen_msgs = rosmsg_generator.genmsg_py( - msg_files=[os.path.join(path, f) for f in os.listdir(path)], # every file not ending in '.msg' will be ignored - package=self.rospackage, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - initpy=True # we always create an __init__.py when called from here. - ) - init_path = None - for pyf in self.gen_msgs: - if pyf.endswith('__init__.py'): - init_path = pyf - elif loader_file_extension == '.srv': - # TODO : dynamic in memory generation (we do not need the file ultimately...) - self.gen_msgs = rosmsg_generator.gensrv_py( - srv_files=[os.path.join(path, f) for f in os.listdir(path)], - # every file not ending in '.msg' will be ignored - package=self.rospackage, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - initpy=True # we always create an __init__.py when called from here. - ) - init_path = None - for pyf in self.gen_msgs: - if pyf.endswith('__init__.py'): - init_path = pyf - else: - raise RuntimeError( - "RosDefLoader for a format {0} other than .msg or .srv is not supported".format( - rosdef_extension)) - - if not init_path: - raise ImportError("__init__.py file not found".format(init_path)) - if not os.path.exists(init_path): - raise ImportError("{0} file not found".format(init_path)) - - # relying on usual source file loader since we have generated normal python code - # BUT we need to pass the directory path (not the init file path like for python3) - super(ROSDefLoader, self).__init__(fullname, os.path.dirname(init_path)) - else: # it is a directory potentially containing an 'msg' - # If we are here, it means it wasn't loaded before - # We need to be able to load from source - super(ROSDefLoader, self).__init__(fullname, path) - - # or to load from installed ros package (python already generated, no point to generate again) - # Note : the path being in sys.path or not is a matter of ROS setup or metafinder. - # TODO - - elif os.path.isfile(path): - # The file should have already been generated (by the loader for a msg package) - # Note we do not want to rely on namespace packages here, since they are not standardized for python2, - # and they can prevent some useful usecases. - # TODO : This seems to be not used. confirm and cleanup - # Hack to be able to "import generated classes" - modname = fullname.rpartition('.')[2] - filepath = os.path.join(self.outdir_pkg, loader_generated_subdir, '_' + modname + '.py') # the generated module - # relying on usual source file loader since we have previously generated normal python code - super(ROSDefLoader, self).__init__(fullname, filepath) - - elif sys.version_info >= (3, 4): # we do not support 3.2 and 3.3 (unsupported but might work ?) - import importlib.machinery - - class ROSDefLoader(importlib.machinery.SourceFileLoader): - def __init__(self, fullname, path): - - self.logger = logging.getLogger(__name__) - - # Doing this in each loader, in case we are running from different processes, - # avoiding to reload from same file (especially useful for boxed tests). - # But deterministic path to avoid regenerating from the same interpreter - self.rosimport_path = os.path.join(tempfile.gettempdir(), 'rosimport', str(os.getpid())) - if os.path.exists(self.rosimport_path): - shutil.rmtree(self.rosimport_path) - os.makedirs(self.rosimport_path) - - self.rospackage = fullname.partition('.')[0] - # We should reproduce package structure in generated file structure - dirlist = path.split(os.sep) - pkgidx = dirlist[::-1].index(self.rospackage) - indirlist = [p for p in dirlist[:len(dirlist)-pkgidx-1:-1] if p != loader_origin_subdir and not p.endswith(loader_file_extension)] - self.outdir_pkg = os.path.join(self.rosimport_path, self.rospackage, *indirlist[::-1]) - - if os.path.isdir(path): - if path.endswith(loader_origin_subdir) and any([f.endswith(loader_file_extension) for f in os.listdir(path)]): # if we get a non empty 'msg' folder - init_path = os.path.join(self.outdir_pkg, loader_generated_subdir, '__init__.py') - if not os.path.exists(init_path): - # TODO : we need to determine that from the loader - # as a minimum we need to add current package - self.includepath = [self.rospackage + ':' + path] - - # TODO : unify this after reviewing rosmsg_generator API - if loader_file_extension == '.msg': - # TODO : dynamic in memory generation (we do not need the file ultimately...) - self.gen_msgs = rosmsg_generator.genmsg_py( - msg_files=[os.path.join(path, f) for f in os.listdir(path)], # every file not ending in '.msg' will be ignored - package=self.rospackage, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - initpy=True # we always create an __init__.py when called from here. - ) - init_path = None - for pyf in self.gen_msgs: - if pyf.endswith('__init__.py'): - init_path = pyf - elif loader_file_extension == '.srv': - # TODO : dynamic in memory generation (we do not need the file ultimately...) - self.gen_msgs = rosmsg_generator.gensrv_py( - srv_files=[os.path.join(path, f) for f in os.listdir(path)], - # every file not ending in '.msg' will be ignored - package=self.rospackage, - outdir_pkg=self.outdir_pkg, - includepath=self.includepath, - initpy=True # we always create an __init__.py when called from here. - ) - init_path = None - for pyf in self.gen_msgs: - if pyf.endswith('__init__.py'): - init_path = pyf - else: - raise RuntimeError( - "RosDefLoader for a format {0} other than .msg or .srv is not supported".format( - rosdef_extension)) - - if not init_path: - raise ImportError("__init__.py file not found".format(init_path)) - if not os.path.exists(init_path): - raise ImportError("{0} file not found".format(init_path)) - - # relying on usual source file loader since we have generated normal python code - super(ROSDefLoader, self).__init__(fullname, init_path) - else: # it is a directory potentially containing an 'msg' - # If we are here, it means it wasn't loaded before - # We need to be able to load from source - super(ROSDefLoader, self).__init__(fullname, path) - - # or to load from installed ros package (python already generated, no point to generate again) - # Note : the path being in sys.path or not is a matter of ROS setup or metafinder. - # TODO - - elif os.path.isfile(path): - # The file should have already been generated (by the loader for a msg package) - # Note we do not want to rely on namespace packages here, since they are not standardized for python2, - # and they can prevent some useful usecases. - - # Hack to be able to "import generated classes" - modname = fullname.rpartition('.')[2] - filepath = os.path.join(self.outdir_pkg, loader_generated_subdir, '_' + modname + '.py') # the generated module - # relying on usual source file loader since we have previously generated normal python code - super(ROSDefLoader, self).__init__(fullname, filepath) - - def get_gen_path(self): - """Returning the generated path matching the import""" - return os.path.join(self.outdir_pkg, loader_generated_subdir) - - def __repr__(self): - return "ROSDefLoader/{0}({1}, {2})".format(loader_file_extension, self.name, self.path) - - else: - raise ImportError("ros_loader : Unsupported python version") - - return ROSDefLoader - -ROSMsgLoader = RosLoader(rosdef_extension='.msg') -ROSSrvLoader = RosLoader(rosdef_extension='.srv') diff --git a/pyros_msgs/importer/rosmsg_metafinder.py b/pyros_msgs/importer/rosmsg_metafinder.py deleted file mode 100644 index 3f7d328..0000000 --- a/pyros_msgs/importer/rosmsg_metafinder.py +++ /dev/null @@ -1,192 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import contextlib -import importlib -import site - -from pyros_msgs.importer import rosmsg_generator - -""" -A module to setup custom importer for .msg and .srv files -Upon import, it will first find the .msg file, then generate the python module for it, then load it. - -TODO... -""" - -# We need to be extra careful with python versions -# Ref : https://docs.python.org/dev/library/importlib.html#importlib.import_module - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path -# Note : Couldn't find a way to make imp.load_source deal with packages or relative imports (necessary for our generated message classes) -import os -import sys - -# This will take the ROS distro version if ros has been setup -import genpy.generator -import genpy.generate_initpy - -import logging - - - - - -if sys.version_info >= (3, 4): - - import importlib.abc - import importlib.machinery - - class ROSImportMetaFinder(importlib.abc.MetaPathFinder): - def __init__(self, *workspaces): - """ - :param workspaces: can be a devel or install workspace (including a distro install directory), but also a directory containing packages (like a source workspace) - These should all work, without catkin build necessary. - """ - super(ROSImportMetaFinder, self).__init__() - # TODO : prevent that when we are in virtualenv and we have not allowed system site packages - - self.src_rospkgs = [] - - broken_workspaces = [] - for w in workspaces: - # Adding the share path (where we can find all packages and their messages) - share_path = os.path.join(w, 'share') - if os.path.exists(share_path): - self.share_path = share_path - else: # this is not a workspace, maybe we are expected to get the package directly from source ? - found_one = False - for root, dirs, files in os.walk(w, topdown=False): - if 'package.xml' in files: # we have found a ros package - self.src_rospkgs.append(root) - found_one = True - if not found_one: - broken_workspaces.append(w) - - raise ImportError - - python_libpath = os.path.join(w, 'lib', 'python' + sys.version_info.major + '.' + sys.version_info.minor) - if os.path.exists(python_libpath): - # adding python site directories for the ROS distro (to find python modules as usual with ROS) - if os.path.exists(os.path.join(python_libpath, 'dist-packages')): - site.addsitedir(os.path.join(python_libpath, 'dist-packages')) - if os.path.exists(os.path.join(python_libpath, 'site-packages')): - site.addsitedir(os.path.join(python_libpath, 'site-packages')) - else: # this is not a workspace, maybe we are expected to get the package directly from source ? - - for root, dirs, files in os.walk(w, topdown=False): - if 'package.xml' in files: # we have found a ros package - self.src_rospkgs.append(root) - - raise ImportError - - - - - def find_spec(self, fullname, path, target=None): - """ - :param fullname: the name of the package we are trying to import - :param path: path we we expect to find it (can be None) - :param target: what we plan to do with it - :return: - """ - # TODO: read PEP 420 :) - last_mile = fullname.split('.')[-1] - - path = path or sys.path - - # TODO: we should handle different structure of ROS packages (installed or not) - #if (os.path.exists) - - # TODO : similar to pyros-setup ? - # + setup rosmsg importers ? - if (not os.path.exists(os.path.join(last_mile, 'package.xml')) or - False): - raise ImportError - - for p in path: - fullpath = os.path.join(p, last_mile) - init_fullpath = os.path.join(fullpath, '__init__.ay') - module_fullpath = fullpath + '.ay' - - if os.path.isdir(fullpath) and os.path.exists(init_fullpath): - return importlib.machinery.ModuleSpec( - fullname, - loader, - origin=init_fullpath - ) - - else: - if os.path.exists(module_fullpath): - return importlib.machinery.ModuleSpec( - fullname, - loader, - origin=module_fullpath - ) - - return None - -else: - passb # TODO - - -# Useless ? -#_ros_finder_instance_obsolete_python = ROSImportFinder - -ros_distro_finder = None - - -def activate(rosdistro_path=None, *workspaces): - global ros_distro_finder - if rosdistro_path is None: # autodetect most recent installed distro - if os.path.exists('/opt/ros/lunar'): - rosdistro_path = '/opt/ros/lunar' - elif os.path.exists('/opt/ros/kinetic'): - rosdistro_path = '/opt/ros/kinetic' - elif os.path.exists('/opt/ros/jade'): - rosdistro_path = '/opt/ros/jade' - elif os.path.exists('/opt/ros/indigo'): - rosdistro_path = '/opt/ros/indigo' - else: - raise ImportError( - "No ROS distro detected on this system. Please specify the path when calling ROSImportMetaFinder()") - - ros_distro_finder = ROSImportMetaFinder(rosdistro_path, *workspaces) - sys.meta_path.append(ros_distro_finder) - - #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? - - # We need to be before FileFinder to be able to find our (non .py[c]) files - # inside, maybe already imported, python packages... - sys.path_hooks.insert(1, ROSImportFinder) - - # else: # older (trusty) version - # sys.path_hooks.append(_ros_finder_instance_obsolete_python) - - for hook in sys.path_hooks: - print('Path hook: {}'.format(hook)) - - # TODO : mix that with ROS PYTHONPATH shenanigans... to enable the finder only for 'ROS aware' paths - if paths: - sys.path.append(*paths) - - -def deactivate(*paths): - """ CAREFUL : even if we remove our path_hooks, the created finder are still cached in sys.path_importer_cache.""" - #if sys.version_info >= (2, 7, 12): # TODO : which exact version matters ? - sys.path_hooks.remove(ROSImportFinder) - # else: # older (trusty) version - # sys.path_hooks.remove(_ros_finder_instance_obsolete_python) - if paths: - sys.path.remove(*paths) - - sys.meta_path.remove(ros_distro_finder) - - -@contextlib.contextmanager -def ROSImportContext(*paths): - activate(*paths) - yield - deactivate(*paths) - - -# TODO : a meta finder could find a full ROS distro... diff --git a/pyros_msgs/importer/tests/__init__.py b/pyros_msgs/importer/tests/__init__.py deleted file mode 100644 index 8b8e09e..0000000 --- a/pyros_msgs/importer/tests/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import absolute_import, division, print_function -# -# import os -# try: -# # Using std_msgs directly if ROS has been setup (while using from ROS pkg) -# import std_msgs -# -# except ImportError: -# -# # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) -# -# import site -# site.addsitedir(os.path.join( -# os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), -# 'ros-site' -# )) -# -# import std_msgs -# -# # Note we do not want to use pyros_setup here. -# # We do not want to do a full ROS setup, only import specific packages. -# # If needed it should have been done before (loading a parent package). -# # this handle the case where we want to be independent of any underlying ROS system. diff --git a/pyros_msgs/importer/tests/_utils.py b/pyros_msgs/importer/tests/_utils.py deleted file mode 100644 index 22c3667..0000000 --- a/pyros_msgs/importer/tests/_utils.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import absolute_import, print_function - - -def print_importers(): - import sys - import pprint - - print('PATH:'), - pprint.pprint(sys.path) - print() - print('IMPORTERS:') - for name, cache_value in sys.path_importer_cache.items(): - name = name.replace(sys.prefix, '...') - print('%s: %r' % (name, cache_value)) diff --git a/pyros_msgs/importer/tests/msg/TestMsg.msg b/pyros_msgs/importer/tests/msg/TestMsg.msg deleted file mode 100644 index 6634a31..0000000 --- a/pyros_msgs/importer/tests/msg/TestMsg.msg +++ /dev/null @@ -1,3 +0,0 @@ -string test_string -bool test_bool -# TOOD : add more fields for more testing \ No newline at end of file diff --git a/pyros_msgs/importer/tests/srv/TestSrv.srv b/pyros_msgs/importer/tests/srv/TestSrv.srv deleted file mode 100644 index 0ae2ee7..0000000 --- a/pyros_msgs/importer/tests/srv/TestSrv.srv +++ /dev/null @@ -1,4 +0,0 @@ -string test_request ---- -bool test_response -# TOOD : add more fields for more testing \ No newline at end of file diff --git a/pyros_msgs/importer/tests/test_rosmsg_generator_exec.py b/pyros_msgs/importer/tests/test_rosmsg_generator_exec.py deleted file mode 100644 index b5d109c..0000000 --- a/pyros_msgs/importer/tests/test_rosmsg_generator_exec.py +++ /dev/null @@ -1,71 +0,0 @@ -from __future__ import absolute_import, division, print_function - -""" -Testing executing rosmsg_generator directly (like setup.py would) -""" - -import os -import runpy - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path - -# Including generator module directly from code to be able to generate our message classes -import imp -rosmsg_generator = imp.load_source('rosmsg_generator', os.path.join(os.path.dirname(os.path.dirname(__file__)), 'rosmsg_generator.py')) - - -def test_generate_msgsrv_nspkg_usable(): - # generating message class - sitedir, generated_msg, generated_srv = rosmsg_generator.generate_msgsrv_nspkg( - [os.path.join(os.path.dirname(__file__), 'msg', 'TestMsg.msg')], - package='test_gen_msgs', - ns_pkg=True, - ) - - # Verify that files exists and are importable - for m in [generated_msg, generated_srv]: - # modules are generated where the file is launched - gen_file = os.path.join(sitedir, *m.split(".")) - assert os.path.exists(gen_file + '.py') or os.path.exists(os.path.join(gen_file, '__init__.py')) - - msg_mod, srv_mod = rosmsg_generator.import_msgsrv(sitedir, generated_msg, generated_srv) - - assert msg_mod is not None - assert srv_mod is not None - -# def test_generate_msgsrv_samepkg_usable(): -# # generating message class -# sitedir, generated_msg, generated_srv = rosmsg_generator.generate_msgsrv_nspkg( -# [os.path.join(os.path.dirname(__file__), 'msg', 'TestMsg.msg')], -# package='test_gen_msgs', -# ns_pkg=True, -# ) -# -# # Verify that files exists and are importable -# for m in [generated_msg, generated_srv]: -# # modules are generated where the file is launched -# gen_file = os.path.join(sitedir, *m.split(".")) -# assert os.path.exists(gen_file + '.py') or os.path.exists(os.path.join(gen_file, '__init__.py')) -# -# msg_mod, srv_mod = rosmsg_generator.import_msgsrv(sitedir, generated_msg, generated_srv) -# -# assert msg_mod is not None -# assert srv_mod is not None - -def test_generate_msgsrv_genpkg_usable(): - # generating message class - sitedir, generated_msg, generated_srv = rosmsg_generator.generate_msgsrv_nspkg( - [os.path.join(os.path.dirname(__file__), 'msg', 'TestMsg.msg')], - package='test_gen_msgs', - ) - - # Verify that files exists and are importable - for m in [generated_msg, generated_srv]: - # modules are generated where the file is launched - gen_file = os.path.join(sitedir, *m.split(".")) - assert os.path.exists(gen_file + '.py') or os.path.exists(os.path.join(gen_file, '__init__.py')) - - msg_mod, srv_mod = rosmsg_generator.import_msgsrv(sitedir, generated_msg, generated_srv) - - assert msg_mod is not None - assert srv_mod is not None \ No newline at end of file diff --git a/pyros_msgs/importer/tests/test_rosmsg_import.py b/pyros_msgs/importer/tests/test_rosmsg_import.py deleted file mode 100644 index 7c7c888..0000000 --- a/pyros_msgs/importer/tests/test_rosmsg_import.py +++ /dev/null @@ -1,280 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import copy - -""" -Testing rosmsg_import with import keyword. -CAREFUL : these tests should run with pytest --boxed in order to avoid polluting each other sys.modules -""" - -import os -import sys -import runpy -import logging.config - -logging.config.dictConfig({ - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - } - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, -}) - -# Since test frameworks (like pytest) play with the import machinery, we cannot use it here... -import unittest - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path - -import importlib - -# Importing importer module -from pyros_msgs.importer import rosmsg_finder - -# importlib -# https://pymotw.com/3/importlib/index.html -# https://pymotw.com/2/importlib/index.html - - -from ._utils import print_importers - - -class TestImportMsg(unittest.TestCase): - - rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') - - @classmethod - def setUpClass(cls): - # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package - supported_loaders = rosmsg_finder._get_supported_ros_loaders() - ros_hook = rosmsg_finder.ROSDirectoryFinder.path_hook(*supported_loaders) - sys.path_hooks.insert(1, ros_hook) - - sys.path.append(cls.rosdeps_path) - - @classmethod - def tearDownClass(cls): - # CAREFUL : Even though we remove the path from sys.path, - # initialized finders will remain in sys.path_importer_cache - sys.path.remove(cls.rosdeps_path) - - def test_import_absolute_msg(self): - print_importers() - - # Verify that files exists and are importable - import std_msgs.msg as std_msgs - - self.assertTrue(std_msgs is not None) - self.assertTrue(std_msgs.Bool is not None) - self.assertTrue(callable(std_msgs.Bool)) - self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - - # use it ! - self.assertTrue(std_msgs.Bool(True)) - - def test_import_class_from_absolute_msg(self): - """Verify that""" - print_importers() - - # Verify that files exists and are importable - from std_msgs.msg import Bool - - self.assertTrue(Bool is not None) - self.assertTrue(callable(Bool)) - self.assertTrue(Bool._type == 'std_msgs/Bool') - - # use it ! - self.assertTrue(Bool(True)) - - def test_import_relative_msg(self): - """Verify that package is importable relatively""" - print_importers() - - from . import msg as test_msgs - - self.assertTrue(test_msgs is not None) - self.assertTrue(test_msgs.TestMsg is not None) - self.assertTrue(callable(test_msgs.TestMsg)) - self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name - - # use it ! - self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) - - def test_import_class_from_relative_msg(self): - """Verify that message class is importable relatively""" - print_importers() - - from .msg import TestMsg - - self.assertTrue(TestMsg is not None) - self.assertTrue(callable(TestMsg)) - self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') - - # use it ! - self.assertTrue(TestMsg(test_bool=True, test_string='Test').test_bool) - - def test_import_absolute_class_raises(self): - print_importers() - - with self.assertRaises(ImportError): - import std_msgs.msg.Bool - - def test_double_import_uses_cache(self): # - print_importers() - # Verify that files exists and are importable - import std_msgs.msg as std_msgs - - self.assertTrue(std_msgs.Bool is not None) - self.assertTrue(callable(std_msgs.Bool)) - self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - - import std_msgs.msg as std_msgs2 - - self.assertTrue(std_msgs == std_msgs2) - - -class TestImportSrv(unittest.TestCase): - - rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps', 'ros_comm_msgs') - - @classmethod - def setUpClass(cls): - # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package - supported_loaders = rosmsg_finder._get_supported_ros_loaders() - ros_hook = rosmsg_finder.ROSDirectoryFinder.path_hook(*supported_loaders) - sys.path_hooks.insert(1, ros_hook) - - sys.path.append(cls.rosdeps_path) - - @classmethod - def tearDownClass(cls): - # CAREFUL : Even though we remove the path from sys.path, - # initialized finders will remain in sys.path_importer_cache - sys.path.remove(cls.rosdeps_path) - - def test_import_absolute_srv(self): - print_importers() - - # Verify that files exists and are importable - import std_srvs.srv as std_srvs - - self.assertTrue(std_srvs is not None) - self.assertTrue(std_srvs.SetBool is not None) - self.assertTrue(callable(std_srvs.SetBool)) - self.assertTrue(std_srvs.SetBool._type == 'std_srvs/SetBool') - - self.assertTrue(std_srvs is not None) - self.assertTrue(std_srvs.SetBoolRequest is not None) - self.assertTrue(callable(std_srvs.SetBoolRequest)) - self.assertTrue(std_srvs.SetBoolRequest._type == 'std_srvs/SetBoolRequest') - - self.assertTrue(std_srvs is not None) - self.assertTrue(std_srvs.SetBoolResponse is not None) - self.assertTrue(callable(std_srvs.SetBoolResponse)) - self.assertTrue(std_srvs.SetBoolResponse._type == 'std_srvs/SetBoolResponse') - - # use it ! - self.assertTrue(std_srvs.SetBoolRequest(data=True).data) - self.assertTrue(std_srvs.SetBoolResponse(success=True, message='Test').success) - - def test_import_class_from_absolute_srv(self): - """Verify that""" - print_importers() - - # Verify that files exists and are importable - from std_srvs.srv import SetBool, SetBoolRequest, SetBoolResponse - - self.assertTrue(SetBool is not None) - self.assertTrue(callable(SetBool)) - self.assertTrue(SetBool._type == 'std_srvs/SetBool') - - self.assertTrue(SetBoolRequest is not None) - self.assertTrue(callable(SetBoolRequest)) - self.assertTrue(SetBoolRequest._type == 'std_srvs/SetBoolRequest') - - self.assertTrue(SetBoolResponse is not None) - self.assertTrue(callable(SetBoolResponse)) - self.assertTrue(SetBoolResponse._type == 'std_srvs/SetBoolResponse') - - # use it ! - self.assertTrue(SetBoolRequest(data=True).data) - self.assertTrue(SetBoolResponse(success=True, message='Test').success) - - def test_import_relative_srv(self): - """Verify that package is importable relatively""" - print_importers() - - from . import srv as test_srvs - - self.assertTrue(test_srvs is not None) - - self.assertTrue(test_srvs.TestSrv is not None) - self.assertTrue(callable(test_srvs.TestSrv)) - self.assertTrue(test_srvs.TestSrv._type == 'pyros_msgs/TestSrv') # careful between ros package name and python package name - - self.assertTrue(test_srvs.TestSrvRequest is not None) - self.assertTrue(callable(test_srvs.TestSrvRequest)) - self.assertTrue(test_srvs.TestSrvRequest._type == 'pyros_msgs/TestSrvRequest') # careful between ros package name and python package name - - self.assertTrue(test_srvs.TestSrvResponse is not None) - self.assertTrue(callable(test_srvs.TestSrvResponse)) - self.assertTrue(test_srvs.TestSrvResponse._type == 'pyros_msgs/TestSrvResponse') # careful between ros package name and python package name - - # use it ! - self.assertTrue(test_srvs.TestSrvRequest(test_request='Test').test_request) - self.assertTrue(test_srvs.TestSrvResponse(test_response=True).test_response) - - def test_import_class_from_relative_srv(self): - """Verify that message class is importable relatively""" - print_importers() - - from .srv import TestSrv, TestSrvRequest, TestSrvResponse - - self.assertTrue(TestSrv is not None) - self.assertTrue(callable(TestSrv)) - self.assertTrue(TestSrv._type == 'pyros_msgs/TestSrv') # careful between ros package name and python package name - - self.assertTrue(TestSrvRequest is not None) - self.assertTrue(callable(TestSrvRequest)) - self.assertTrue(TestSrvRequest._type == 'pyros_msgs/TestSrvRequest') - - self.assertTrue(TestSrvResponse is not None) - self.assertTrue(callable(TestSrvResponse)) - self.assertTrue(TestSrvResponse._type == 'pyros_msgs/TestSrvResponse') - - # use it ! - self.assertTrue(TestSrvRequest(test_request='Test').test_request) - self.assertTrue(TestSrvResponse(test_response=True).test_response) - - def test_import_absolute_class_raises(self): - print_importers() - - with self.assertRaises(ImportError): - import std_srvs.srv.SetBool - - def test_double_import_uses_cache(self): # - print_importers() - # Verify that files exists and are importable - import std_srvs.srv as std_srvs - - self.assertTrue(std_srvs.SetBool is not None) - self.assertTrue(std_srvs.SetBoolRequest is not None) - self.assertTrue(std_srvs.SetBoolResponse is not None) - - import std_srvs.srv as std_srvs2 - - self.assertTrue(std_srvs == std_srvs2) - -if __name__ == '__main__': - import pytest - pytest.main(['-s', '-x', __file__, '--boxed']) diff --git a/pyros_msgs/importer/tests/test_rosmsg_importlib.py b/pyros_msgs/importer/tests/test_rosmsg_importlib.py deleted file mode 100644 index 87330c7..0000000 --- a/pyros_msgs/importer/tests/test_rosmsg_importlib.py +++ /dev/null @@ -1,314 +0,0 @@ -from __future__ import absolute_import, division, print_function -""" -Testing dynamic import with importlib -""" - -import os -import sys -import runpy -import logging.config - -logging.config.dictConfig({ - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - } - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, -}) - -# Relying on basic unittest first, to be able to easily switch the test framework in case of import conflicts. -import unittest - -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path - -import importlib - -# Importing importer module -from pyros_msgs.importer import rosmsg_finder - -# importlib -# https://pymotw.com/3/importlib/index.html -# https://pymotw.com/2/importlib/index.html - -# -# Note : we cannot assume anything about import implementation (different python version, different version of pytest) -# => we need to test them all... -# - - -from ._utils import print_importers - - -class TestImportLibAnotherMsg(unittest.TestCase): - rosdeps_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') - - @classmethod - def setUpClass(cls): - # We need to be before FileFinder to be able to find our '.msg' and '.srv' files without making a namespace package - supported_loaders = rosmsg_finder._get_supported_ros_loaders() - ros_hook = rosmsg_finder.ROSDirectoryFinder.path_hook(*supported_loaders) - sys.path_hooks.insert(1, ros_hook) - - sys.path.append(cls.rosdeps_path) - - @classmethod - def tearDownClass(cls): - # CAREFUL : Even though we remove the path from sys.path, - # initialized finders will remain in sys.path_importer_cache - sys.path.remove(cls.rosdeps_path) - - @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_absolute_msg(self): - # Verify that files exists and are importable - std_msgs = importlib.__import__('std_msgs.msg') - std_msgs = std_msgs.msg - - self.assertTrue(std_msgs is not None) - self.assertTrue(std_msgs.Bool is not None) - self.assertTrue(callable(std_msgs.Bool)) - self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - - # use it ! - self.assertTrue(std_msgs.Bool(True)) - - @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_absolute_class_raises(self): - with self.assertRaises(ImportError): - importlib.__import__('std_msgs.msg.Bool') - - # BROKEN 3.4 ? - @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_relative_pkg(self): - # Verify that files exists and are importable - test_msgs = importlib.__import__('msg', globals=globals(), level=1) - - self.assertTrue(test_msgs is not None) - self.assertTrue(test_msgs.TestMsg is not None) - self.assertTrue(callable(test_msgs.TestMsg)) - self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name - - # use it ! - self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) - - # BROKEN 3.4 ? - @unittest.skipIf(not hasattr(importlib, '__import__'), reason="importlib does not have attribute __import__") - def test_importlib_import_relative_mod(self): - # Verify that files exists and are importable - msg = importlib.__import__('msg.TestMsg', globals=globals(), level=1) - TestMsg = msg.TestMsg - - self.assertTrue(TestMsg is not None) - self.assertTrue(callable(TestMsg)) - self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name - - # use it ! - self.assertTrue(TestMsg(test_bool=True, test_string='Test').test_bool) - - @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), - reason="importlib does not have attribute find_loader or load_module") - def test_importlib_loadmodule_absolute_msg(self): - # Verify that files exists and are dynamically importable - pkg_list = 'std_msgs.msg'.split('.')[:-1] - mod_list = 'std_msgs.msg'.split('.')[1:] - pkg = None - for pkg_name, mod_name in zip(pkg_list, mod_list): - pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) - pkg = pkg_loader.load_module(mod_name) - - std_msgs = pkg - - self.assertTrue(std_msgs is not None) - self.assertTrue(std_msgs.Bool is not None) - self.assertTrue(callable(std_msgs.Bool)) - self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - - # use it ! - self.assertTrue(std_msgs.Bool(True)) - - # TODO : implement some differences and check we get them... - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(std_msgs) - else: - pass - - @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), - reason="importlib does not have attribute find_loader or load_module") - def test_importlib_loadmodule_absolute_class(self): - # Verify that files exists and are dynamically importable - pkg_list = 'std_msgs.msg.Bool'.split('.')[:-1] - mod_list = 'std_msgs.msg.Bool'.split('.')[1:] - pkg = None - for pkg_name, mod_name in zip(pkg_list, mod_list): - pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) - pkg = pkg_loader.load_module(mod_name) - - Bool = pkg - - self.assertTrue(Bool is not None) - self.assertTrue(callable(Bool)) - self.assertTrue(Bool._type == 'std_msgs/Bool') - - # use it ! - self.assertTrue(Bool(True)) - - # TODO : implement some differences and check we get them... - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(Bool) - else: - pass - - @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), - reason="importlib does not have attribute find_loader or load_module") - def test_importlib_loadmodule_relative_msg(self): - # Verify that files exists and are dynamically importable - pkg_list = '.msg'.split('.')[:-1] - mod_list = '.msg'.split('.')[1:] - pkg = None - for pkg_name, mod_name in zip(pkg_list, mod_list): - pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) - pkg = pkg_loader.load_module(mod_name) - - test_msgs = pkg - - self.assertTrue(test_msgs is not None) - self.assertTrue(test_msgs.TestMsg is not None) - self.assertTrue(callable(test_msgs.TestMsg)) - self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name - - # use it ! - self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) - - # TODO : implement some differences and check we get them... - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(test_msgs) - else: - pass - - @unittest.skipIf(not hasattr(importlib, 'find_loader') or not hasattr(importlib, 'load_module'), - reason="importlib does not have attribute find_loader or load_module") - def test_importlib_loadmodule_relative_class(self): - # Verify that files exists and are dynamically importable - pkg_list = '.msg.TestMsg'.split('.')[:-1] - mod_list = '.msg.TestMsg'.split('.')[1:] - pkg = None - for pkg_name, mod_name in zip(pkg_list, mod_list): - pkg_loader = importlib.find_loader(pkg_name, pkg.__path__ if pkg else None) - pkg = pkg_loader.load_module(mod_name) - - TestMsg = pkg - - self.assertTrue(TestMsg is not None) - self.assertTrue(callable(TestMsg)) - self.assertTrue(TestMsg._type == 'pyros_msgs/TestMsg') - - # use it ! - self.assertTrue(TestMsg(test_bool=True, test_string='Test').test_bool) - - # TODO : implement some differences and check we get them... - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(TestMsg) - else: - pass - - # TODO : dynamic using module_spec (python 3.5) - - @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") - def test_importlib_importmodule_absolute_msg(self): - # Verify that files exists and are dynamically importable - std_msgs = importlib.import_module('std_msgs.msg') - - self.assertTrue(std_msgs is not None) - self.assertTrue(std_msgs.Bool is not None) - self.assertTrue(callable(std_msgs.Bool)) - self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - - # use it ! - self.assertTrue(std_msgs.Bool(True)) - - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(std_msgs) - else: - pass - - assert std_msgs is not None - - @unittest.skipIf(not hasattr(importlib, 'import_module'), - reason="importlib does not have attribute import_module") - def test_importlib_importmodule_absolute_class_raises(self): - with self.assertRaises(ImportError): - importlib.import_module('std_msgs.msg.Bool') - - @unittest.skipIf(not hasattr(importlib, 'import_module'), reason="importlib does not have attribute import_module") - def test_importlib_importmodule_relative_msg(self): - - assert __package__ - # Verify that files exists and are dynamically importable - test_msgs = importlib.import_module('.msg', package=__package__) - - self.assertTrue(test_msgs is not None) - self.assertTrue(test_msgs.TestMsg is not None) - self.assertTrue(callable(test_msgs.TestMsg)) - self.assertTrue(test_msgs.TestMsg._type == 'pyros_msgs/TestMsg') # careful between ros package name and python package name - - # use it ! - self.assertTrue(test_msgs.TestMsg(test_bool=True, test_string='Test').test_bool) - - if hasattr(importlib, 'reload'): # recent version of importlib - # attempting to reload - importlib.reload(test_msgs) - else: - pass - - assert test_msgs is not None - - @unittest.skipIf(not hasattr(importlib, 'import_module'), - reason="importlib does not have attribute import_module") - def test_importlib_importmodule_relative_class_raises(self): - - assert __package__ - with self.assertRaises(ImportError): - importlib.import_module('.msg.TestMsg', package=__package__) - - # TODO - # def test_double_import_uses_cache(self): # - # print_importers() - # # Verify that files exists and are importable - # import std_msgs.msg as std_msgs - # - # self.assertTrue(std_msgs.Bool is not None) - # self.assertTrue(callable(std_msgs.Bool)) - # self.assertTrue(std_msgs.Bool._type == 'std_msgs/Bool') - # - # import std_msgs.msg as std_msgs2 - # - # self.assertTrue(std_msgs == std_msgs2) - - - - -def test_importlib_srv_module(): - pass - # TODO - # # Verify that files exists and are importable - # msg_mod = importlib.import_module('test_gen_msgs.srv.TestSrv') - - -if __name__ == '__main__': - import pytest - pytest.main(['-s', '-x', __file__, '--boxed']) diff --git a/pyros_msgs/opt_as_array/tests/test_opt_bool.py b/pyros_msgs/opt_as_array/tests/test_opt_bool.py index a877a66..706650f 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_bool.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_bool.py @@ -2,11 +2,15 @@ import os import sys +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -test_gen_msgs, test_gen_srvs = msg_generate.generate_test_msgs() +import rosimport +rosimport.activate() + + +from . import msg as test_gen_msgs # patching (need to know the field name) import pyros_msgs.opt_as_array @@ -14,8 +18,6 @@ import pytest - -import hypothesis import hypothesis.strategies @@ -56,7 +58,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_bool_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of" in str(cm.value) # Just in case we run this directly diff --git a/pyros_msgs/opt_as_array/tests/test_opt_duration.py b/pyros_msgs/opt_as_array/tests/test_opt_duration.py index 17405ca..252bdf4 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_duration.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_duration.py @@ -5,19 +5,19 @@ import genpy +import site -# TODO : find a better place for this ? +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -from pyros_msgs.importer.rosmsg_generator import generate_msgsrv_nspkg, import_msgsrv +import rosimport +rosimport.activate() +from . import msg as test_gen_msgs -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_duration_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_duration_as_array, ['data']) import pytest import hypothesis @@ -38,7 +38,7 @@ ), max_size=1 )) def test_init_rosdata(data): - msg = gen_test_msgs.test_opt_duration_as_array(data=data) + msg = test_gen_msgs.test_opt_duration_as_array(data=data) assert msg.data == data @@ -57,7 +57,7 @@ def test_init_rosdata(data): ) ) def test_init_data(data): - msg = gen_test_msgs.test_opt_duration_as_array(data=data) + msg = test_gen_msgs.test_opt_duration_as_array(data=data) assert msg.data == [data] @@ -75,12 +75,12 @@ def test_init_data(data): ) ) def test_init_raw(data): - msg = gen_test_msgs.test_opt_duration_as_array(data) + msg = test_gen_msgs.test_opt_duration_as_array(data) assert msg.data == [data] def test_init_default(): - msg = gen_test_msgs.test_opt_duration_as_array() + msg = test_gen_msgs.test_opt_duration_as_array() assert msg.data == [] # TODO : test_wrong_init_except(data) check typecheck.tests.test_ros_mappings diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int16.py b/pyros_msgs/opt_as_array/tests/test_opt_int16.py index 83fad1a..9906c46 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_int16.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_int16.py @@ -3,14 +3,19 @@ import os import sys +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_int16_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_int16_as_array, ['data']) import pytest import hypothesis @@ -20,27 +25,27 @@ @hypothesis.given(hypothesis.strategies.lists(hypothesis.strategies.integers(min_value=-32768, max_value=32767), max_size=1)) def test_init_rosdata(data): """Testing that a proper data is stored as is""" - msg = gen_test_msgs.test_opt_int16_as_array(data=data) + msg = test_gen_msgs.test_opt_int16_as_array(data=data) assert msg.data == data @hypothesis.given(hypothesis.strategies.integers(min_value=-32768, max_value=32767)) def test_init_data(data): """Testing that an implicitely convertible data is stored as expected""" - msg = gen_test_msgs.test_opt_int16_as_array(data=data) + msg = test_gen_msgs.test_opt_int16_as_array(data=data) assert msg.data == [data] @hypothesis.given(hypothesis.strategies.integers(min_value=-32768, max_value=32767)) def test_init_raw(data): """Testing storing of data without specifying the field""" - msg = gen_test_msgs.test_opt_int16_as_array(data) + msg = test_gen_msgs.test_opt_int16_as_array(data) assert msg.data == [data] def test_init_default(): """Testing default value""" - msg = gen_test_msgs.test_opt_int16_as_array() + msg = test_gen_msgs.test_opt_int16_as_array() assert msg.data == [] @@ -53,9 +58,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_int16_as_array(data) + test_gen_msgs.test_opt_int16_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int32.py b/pyros_msgs/opt_as_array/tests/test_opt_int32.py index 5c0a7f9..78cc4b3 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_int32.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_int32.py @@ -3,14 +3,19 @@ import os import sys +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_int32_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_int32_as_array, ['data']) import pytest import hypothesis @@ -20,27 +25,27 @@ @hypothesis.given(hypothesis.strategies.lists(hypothesis.strategies.integers(min_value=-2147483648, max_value=2147483647),max_size=1)) def test_init_rosdata(data): """Testing that a proper data is stored as is""" - msg = gen_test_msgs.test_opt_int32_as_array(data=data) + msg = test_gen_msgs.test_opt_int32_as_array(data=data) assert msg.data == data @hypothesis.given(hypothesis.strategies.integers(min_value=-2147483648, max_value=2147483647)) def test_init_data(data): """Testing that an implicitely convertible data is stored as expected""" - msg = gen_test_msgs.test_opt_int32_as_array(data=data) + msg = test_gen_msgs.test_opt_int32_as_array(data=data) assert msg.data == [data] @hypothesis.given(hypothesis.strategies.integers(min_value=-2147483648, max_value=2147483647)) def test_init_raw(data): """Testing storing of data without specifying the field""" - msg = gen_test_msgs.test_opt_int32_as_array(data) + msg = test_gen_msgs.test_opt_int32_as_array(data) assert msg.data == [data] def test_init_default(): """Testing default value""" - msg = gen_test_msgs.test_opt_int32_as_array() + msg = test_gen_msgs.test_opt_int32_as_array() assert msg.data == [] @@ -53,9 +58,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_int32_as_array(data) + test_gen_msgs.test_opt_int32_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int64.py b/pyros_msgs/opt_as_array/tests/test_opt_int64.py index 23f9c2c..7d4c21f 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_int64.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_int64.py @@ -2,15 +2,19 @@ import os import sys +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_int64_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_int64_as_array, ['data']) import pytest import hypothesis @@ -21,24 +25,24 @@ @hypothesis.given(hypothesis.strategies.lists(hypothesis.strategies.integers(min_value=six_long(-9223372036854775808), max_value=six_long(9223372036854775807)), max_size=1)) def test_init_rosdata(data): - msg = gen_test_msgs.test_opt_int64_as_array(data=data) + msg = test_gen_msgs.test_opt_int64_as_array(data=data) assert msg.data == data @hypothesis.given(hypothesis.strategies.integers(min_value=six_long(-9223372036854775808), max_value=six_long(9223372036854775807))) def test_init_data(data): - msg = gen_test_msgs.test_opt_int64_as_array(data=data) + msg = test_gen_msgs.test_opt_int64_as_array(data=data) assert msg.data == [data] @hypothesis.given(hypothesis.strategies.integers(min_value=six_long(-9223372036854775808), max_value=six_long(9223372036854775807))) def test_init_raw(data): - msg = gen_test_msgs.test_opt_int64_as_array(data) + msg = test_gen_msgs.test_opt_int64_as_array(data) assert msg.data == [data] def test_init_default(): - msg = gen_test_msgs.test_opt_int64_as_array() + msg = test_gen_msgs.test_opt_int64_as_array() assert msg.data == [] @@ -51,9 +55,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_int64_as_array(data) + test_gen_msgs.test_opt_int64_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int8.py b/pyros_msgs/opt_as_array/tests/test_opt_int8.py index 2f89161..3226707 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_int8.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_int8.py @@ -1,13 +1,19 @@ from __future__ import absolute_import, division, print_function +import os +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_int8_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_int8_as_array, ['data']) import pytest import hypothesis @@ -17,27 +23,27 @@ @hypothesis.given(hypothesis.strategies.lists(hypothesis.strategies.integers(min_value=-128, max_value=127), max_size=1)) def test_init_rosdata(data): """Testing that a proper data is stored as is""" - msg = gen_test_msgs.test_opt_int8_as_array(data=data) + msg = test_gen_msgs.test_opt_int8_as_array(data=data) assert msg.data == data @hypothesis.given(hypothesis.strategies.integers(min_value=-128, max_value=127)) def test_init_data(data): """Testing that an implicitely convertible data is stored as expected""" - msg = gen_test_msgs.test_opt_int8_as_array(data=data) + msg = test_gen_msgs.test_opt_int8_as_array(data=data) assert msg.data == [data] @hypothesis.given(hypothesis.strategies.integers(min_value=-128, max_value=127)) def test_init_raw(data): """Testing storing of data without specifying the field""" - msg = gen_test_msgs.test_opt_int8_as_array(data) + msg = test_gen_msgs.test_opt_int8_as_array(data) assert msg.data == [data] def test_init_default(): """Testing default value""" - msg = gen_test_msgs.test_opt_int8_as_array() + msg = test_gen_msgs.test_opt_int8_as_array() assert msg.data == [] @@ -50,9 +56,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_int8_as_array(data) + test_gen_msgs.test_opt_int8_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py b/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py index 052608b..f699821 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py @@ -5,18 +5,15 @@ import pytest +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -# generating all and accessing the required message class. -from pyros_msgs.opt_as_array.tests import msg_generate +import rosimport +rosimport.activate() -try: - # This should succeed if the message class was already generated - import std_msgs.msg as std_msgs -except ImportError: # we should enter here if the message was not generated yet. - std_msgs, _ = msg_generate.generate_std_msgs() - -test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() +from . import msg as test_gen_msgs +import std_msgs.msg as std_msgs import pyros_msgs.opt_as_array diff --git a/pyros_msgs/opt_as_array/tests/test_opt_string.py b/pyros_msgs/opt_as_array/tests/test_opt_string.py index 1732a35..e4f9135 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_string.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_string.py @@ -3,15 +3,19 @@ import os import sys +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message class. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_string_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_string_as_array, ['data']) import pytest import hypothesis @@ -25,7 +29,7 @@ # hypothesis.strategies.binary() ), max_size=1)) def test_init_rosdata(data): - msg = gen_test_msgs.test_opt_string_as_array(data=data) + msg = test_gen_msgs.test_opt_string_as_array(data=data) assert msg.data == data @@ -36,7 +40,7 @@ def test_init_rosdata(data): # hypothesis.strategies.binary() )) def test_init_data(data): - msg = gen_test_msgs.test_opt_string_as_array(data=data) + msg = test_gen_msgs.test_opt_string_as_array(data=data) assert msg.data == [data] @@ -47,12 +51,12 @@ def test_init_data(data): # hypothesis.strategies.binary() )) def test_init_raw(data): - msg = gen_test_msgs.test_opt_string_as_array(data) + msg = test_gen_msgs.test_opt_string_as_array(data) assert msg.data == [data] def test_init_default(): - msg = gen_test_msgs.test_opt_string_as_array() + msg = test_gen_msgs.test_opt_string_as_array() assert msg.data == [] @@ -66,9 +70,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_string_as_array(data) + test_gen_msgs.test_opt_string_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly diff --git a/pyros_msgs/opt_as_array/tests/test_opt_time.py b/pyros_msgs/opt_as_array/tests/test_opt_time.py index 17eea7e..018d39f 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_time.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_time.py @@ -4,16 +4,20 @@ import sys import genpy +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message class. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_time_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_time_as_array, ['data']) import pytest import hypothesis @@ -28,7 +32,7 @@ ), max_size=1 )) def test_init_rosdata(data): - msg = gen_test_msgs.test_opt_time_as_array(data=data) + msg = test_gen_msgs.test_opt_time_as_array(data=data) assert msg.data == data @@ -40,7 +44,7 @@ def test_init_rosdata(data): ) ) def test_init_data(data): - msg = gen_test_msgs.test_opt_time_as_array(data=data) + msg = test_gen_msgs.test_opt_time_as_array(data=data) assert msg.data == [data] @@ -52,12 +56,12 @@ def test_init_data(data): ) ) def test_init_raw(data): - msg = gen_test_msgs.test_opt_time_as_array(data) + msg = test_gen_msgs.test_opt_time_as_array(data) assert msg.data == [data] def test_init_default(): - msg = gen_test_msgs.test_opt_time_as_array() + msg = test_gen_msgs.test_opt_time_as_array() assert msg.data == [] diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint16.py b/pyros_msgs/opt_as_array/tests/test_opt_uint16.py index a362aa8..7f59679 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_uint16.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_uint16.py @@ -1,13 +1,19 @@ from __future__ import absolute_import, division, print_function +import os +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_uint16_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_uint16_as_array, ['data']) import pytest import hypothesis @@ -17,27 +23,27 @@ @hypothesis.given(hypothesis.strategies.lists(hypothesis.strategies.integers(min_value=0, max_value=65535), max_size=1)) def test_init_rosdata(data): """Testing that a proper data is stored as is""" - msg = gen_test_msgs.test_opt_uint16_as_array(data=data) + msg = test_gen_msgs.test_opt_uint16_as_array(data=data) assert msg.data == data @hypothesis.given(hypothesis.strategies.integers(min_value=0, max_value=65535)) def test_init_data(data): """Testing that an implicitely convertible data is stored as expected""" - msg = gen_test_msgs.test_opt_uint16_as_array(data=data) + msg = test_gen_msgs.test_opt_uint16_as_array(data=data) assert msg.data == [data] @hypothesis.given(hypothesis.strategies.integers(min_value=0, max_value=65535)) def test_init_raw(data): """Testing storing of data without specifying the field""" - msg = gen_test_msgs.test_opt_uint16_as_array(data) + msg = test_gen_msgs.test_opt_uint16_as_array(data) assert msg.data == [data] def test_init_default(): """Testing default value""" - msg = gen_test_msgs.test_opt_uint16_as_array() + msg = test_gen_msgs.test_opt_uint16_as_array() assert msg.data == [] @@ -50,9 +56,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_uint16_as_array(data) + test_gen_msgs.test_opt_uint16_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint32.py b/pyros_msgs/opt_as_array/tests/test_opt_uint32.py index 8655b75..8be00e8 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_uint32.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_uint32.py @@ -1,13 +1,18 @@ from __future__ import absolute_import, division, print_function +import os +import site -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_uint32_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_uint32_as_array, ['data']) import pytest import hypothesis @@ -17,27 +22,27 @@ @hypothesis.given(hypothesis.strategies.lists(hypothesis.strategies.integers(min_value=0, max_value=4294967295), max_size=1)) def test_init_rosdata(data): """Testing that a proper data is stored as is""" - msg = gen_test_msgs.test_opt_uint32_as_array(data=data) + msg = test_gen_msgs.test_opt_uint32_as_array(data=data) assert msg.data == data @hypothesis.given(hypothesis.strategies.integers(min_value=0, max_value=4294967295)) def test_init_data(data): """Testing that an implicitely convertible data is stored as expected""" - msg = gen_test_msgs.test_opt_uint32_as_array(data=data) + msg = test_gen_msgs.test_opt_uint32_as_array(data=data) assert msg.data == [data] @hypothesis.given(hypothesis.strategies.integers(min_value=0, max_value=4294967295)) def test_init_raw(data): """Testing storing of data without specifying the field""" - msg = gen_test_msgs.test_opt_uint32_as_array(data) + msg = test_gen_msgs.test_opt_uint32_as_array(data) assert msg.data == [data] def test_init_default(): """Testing default value""" - msg = gen_test_msgs.test_opt_uint32_as_array() + msg = test_gen_msgs.test_opt_uint32_as_array() assert msg.data == [] @@ -50,9 +55,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_uint32_as_array(data) + test_gen_msgs.test_opt_uint32_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint64.py b/pyros_msgs/opt_as_array/tests/test_opt_uint64.py index f667725..7277854 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_uint64.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_uint64.py @@ -1,13 +1,19 @@ from __future__ import absolute_import, division, print_function +import os +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_uint64_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_uint64_as_array, ['data']) import pytest import hypothesis @@ -17,27 +23,27 @@ @hypothesis.given(hypothesis.strategies.lists(hypothesis.strategies.integers(min_value=0, max_value=18446744073709551615), max_size=1)) def test_init_rosdata(data): """Testing that a proper data is stored as is""" - msg = gen_test_msgs.test_opt_uint64_as_array(data=data) + msg = test_gen_msgs.test_opt_uint64_as_array(data=data) assert msg.data == data @hypothesis.given(hypothesis.strategies.integers(min_value=0, max_value=18446744073709551615)) def test_init_data(data): """Testing that an implicitely convertible data is stored as expected""" - msg = gen_test_msgs.test_opt_uint64_as_array(data=data) + msg = test_gen_msgs.test_opt_uint64_as_array(data=data) assert msg.data == [data] @hypothesis.given(hypothesis.strategies.integers(min_value=0, max_value=18446744073709551615)) def test_init_raw(data): """Testing storing of data without specifying the field""" - msg = gen_test_msgs.test_opt_uint64_as_array(data) + msg = test_gen_msgs.test_opt_uint64_as_array(data) assert msg.data == [data] def test_init_default(): """Testing default value""" - msg = gen_test_msgs.test_opt_uint64_as_array() + msg = test_gen_msgs.test_opt_uint64_as_array() assert msg.data == [] @@ -50,9 +56,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_uint64_as_array(data) + test_gen_msgs.test_opt_uint64_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint8.py b/pyros_msgs/opt_as_array/tests/test_opt_uint8.py index d4ce481..df9daad 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_uint8.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_uint8.py @@ -1,13 +1,19 @@ from __future__ import absolute_import, division, print_function +import os +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message classe. -from pyros_msgs.opt_as_array.tests import msg_generate -gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() import pyros_msgs.opt_as_array # patching (need to know the field name) -pyros_msgs.opt_as_array.duck_punch(gen_test_msgs.test_opt_uint8_as_array, ['data']) +pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_uint8_as_array, ['data']) import pytest import hypothesis @@ -17,27 +23,27 @@ @hypothesis.given(hypothesis.strategies.lists(hypothesis.strategies.integers(min_value=0, max_value=255), max_size=1)) def test_init_rosdata(data): """Testing that a proper data is stored as is""" - msg = gen_test_msgs.test_opt_uint8_as_array(data=data) + msg = test_gen_msgs.test_opt_uint8_as_array(data=data) assert msg.data == data @hypothesis.given(hypothesis.strategies.integers(min_value=0, max_value=255)) def test_init_data(data): """Testing that an implicitely convertible data is stored as expected""" - msg = gen_test_msgs.test_opt_uint8_as_array(data=data) + msg = test_gen_msgs.test_opt_uint8_as_array(data=data) assert msg.data == [data] @hypothesis.given(hypothesis.strategies.integers(min_value=0, max_value=255)) def test_init_raw(data): """Testing storing of data without specifying the field""" - msg = gen_test_msgs.test_opt_uint8_as_array(data) + msg = test_gen_msgs.test_opt_uint8_as_array(data) assert msg.data == [data] def test_init_default(): """Testing default value""" - msg = gen_test_msgs.test_opt_uint8_as_array() + msg = test_gen_msgs.test_opt_uint8_as_array() assert msg.data == [] @@ -50,9 +56,9 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" with pytest.raises(AttributeError) as cm: - gen_test_msgs.test_opt_uint8_as_array(data) + test_gen_msgs.test_opt_uint8_as_array(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/pyros_msgs/opt_as_nested/tests/msg_generate.py b/pyros_msgs/opt_as_nested/tests/msg_generate.py index f156728..3b2dbd7 100644 --- a/pyros_msgs/opt_as_nested/tests/msg_generate.py +++ b/pyros_msgs/opt_as_nested/tests/msg_generate.py @@ -8,74 +8,74 @@ We need to generate all and import only one time ( in case we have one instance of pytest running multiple tests ) """ -from pyros_msgs.importer.rosmsg_generator import generate_msgsrv_nspkg, import_msgsrv - # These depends on file structure and should no be in functions # dependencies for our generated messages -pyros_msgs_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'msg') -std_msgs_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps', 'std_msgs', 'msg') +pyros_msgs_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +rosdeps_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') # our own test messages we need to generate test_gen_msg_dir = os.path.join(os.path.dirname(__file__), 'msg') - -# TODO : replace this by a clever custom importer -def generate_pyros_msgs(): - flist = os.listdir(pyros_msgs_dir) - #flist = flist + os.listdir(std_msgs_dir) - generated = generate_msgsrv_nspkg( - [os.path.join(pyros_msgs_dir, f) for f in flist], - package='pyros_msgs', - dependencies=['pyros_msgs'], - include_path=['pyros_msgs:{0}'.format(pyros_msgs_dir)], - ns_pkg=True - ) - pyros_msgs_msg, pyros_srvs_srv = import_msgsrv(*generated) - - return pyros_msgs_msg, pyros_srvs_srv - -def generate_std_msgs(): - flist = os.listdir(std_msgs_dir) - generated = generate_msgsrv_nspkg( - [os.path.join(std_msgs_dir, f) for f in flist], - package='std_msgs', - dependencies=['std_msgs'], - include_path=['std_msgs:{0}'.format(std_msgs_dir)], - ns_pkg=True - ) - std_msgs, std_srvs = import_msgsrv(*generated) - - return std_msgs, std_srvs - - -def generate_test_msgs(): - try: - # This should succeed if the message has been generated previously (or accessing ROS generated message) - import pyros_msgs.msg as pyros_msgs - except ImportError: # we should enter here if the message class hasn't been generated yet. - pyros_msgs_msg, pyros_msgs_srv = generate_pyros_msgs() - - try: - # This should succeed if the message class was already generated - import std_msgs.msg as std_msgs - except ImportError: # we should enter here if the message was not generated yet. - _, _ = generate_std_msgs() - - flist = os.listdir(test_gen_msg_dir) - #flist = flist + os.listdir(std_msgs_dir) - generated = generate_msgsrv_nspkg( - [os.path.join(test_gen_msg_dir, f) for f in flist], - package='test_nested_gen_msgs', - dependencies=['pyros_msgs', 'std_msgs'], - include_path=['pyros_msgs:{0}'.format(pyros_msgs_dir), 'std_msgs:{0}'.format(std_msgs_dir)], - ns_pkg=True - ) - test_gen_msgs, test_gen_srvs = import_msgsrv(*generated) - - return test_gen_msgs, test_gen_srvs +import rosimport +rosimport.activate_hook_for(rosdeps_dir) + +# # TODO : replace this by a clever custom importer +# def generate_pyros_msgs(): +# flist = os.listdir(pyros_msgs_dir) +# #flist = flist + os.listdir(std_msgs_dir) +# generated = generate_msgsrv_nspkg( +# [os.path.join(pyros_msgs_dir, f) for f in flist], +# package='pyros_msgs', +# dependencies=['pyros_msgs'], +# include_path=['pyros_msgs:{0}'.format(pyros_msgs_dir)], +# ns_pkg=True +# ) +# pyros_msgs_msg, pyros_srvs_srv = import_msgsrv(*generated) +# +# return pyros_msgs_msg, pyros_srvs_srv +# +# def generate_std_msgs(): +# flist = os.listdir(std_msgs_dir) +# generated = generate_msgsrv_nspkg( +# [os.path.join(std_msgs_dir, f) for f in flist], +# package='std_msgs', +# dependencies=['std_msgs'], +# include_path=['std_msgs:{0}'.format(std_msgs_dir)], +# ns_pkg=True +# ) +# std_msgs, std_srvs = import_msgsrv(*generated) +# +# return std_msgs, std_srvs +# +# +# def generate_test_msgs(): +# try: +# # This should succeed if the message has been generated previously (or accessing ROS generated message) +# import pyros_msgs.msg as pyros_msgs +# except ImportError: # we should enter here if the message class hasn't been generated yet. +# pyros_msgs_msg, pyros_msgs_srv = generate_pyros_msgs() +# +# try: +# # This should succeed if the message class was already generated +# import std_msgs.msg as std_msgs +# except ImportError: # we should enter here if the message was not generated yet. +# _, _ = generate_std_msgs() +# +# flist = os.listdir(test_gen_msg_dir) +# #flist = flist + os.listdir(std_msgs_dir) +# generated = generate_msgsrv_nspkg( +# [os.path.join(test_gen_msg_dir, f) for f in flist], +# package='test_nested_gen_msgs', +# dependencies=['pyros_msgs', 'std_msgs'], +# include_path=['pyros_msgs:{0}'.format(pyros_msgs_dir), 'std_msgs:{0}'.format(std_msgs_dir)], +# ns_pkg=True +# ) +# test_gen_msgs, test_gen_srvs = import_msgsrv(*generated) +# +# return test_gen_msgs, test_gen_srvs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_bool.py b/pyros_msgs/opt_as_nested/tests/test_opt_bool.py index 3df1cee..6ba2caf 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_bool.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_bool.py @@ -5,31 +5,19 @@ import os import sys import pytest +import site -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() - -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) +import rosimport +rosimport.activate() +from . import msg as test_gen_msgs import pyros_msgs.opt_as_nested # patching (need to know the field name) pyros_msgs.opt_as_nested.duck_punch(test_gen_msgs.test_opt_bool_as_nested, ['data']) - - - - - import hypothesis import hypothesis.strategies diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_duration.py b/pyros_msgs/opt_as_nested/tests/test_opt_duration.py index 922e494..d10454d 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_duration.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_duration.py @@ -7,19 +7,15 @@ import genpy import pytest -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate - -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() - -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs + import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int16.py b/pyros_msgs/opt_as_nested/tests/test_opt_int16.py index 95fa1d2..4a10a55 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_int16.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_int16.py @@ -5,21 +5,15 @@ import os import sys import pytest +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +import rosimport +rosimport.activate() -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() +from . import msg as test_gen_msgs -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int32.py b/pyros_msgs/opt_as_nested/tests/test_opt_int32.py index e8f5f93..75d584e 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_int32.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_int32.py @@ -5,20 +5,15 @@ import os import sys import pytest +import site -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int64.py b/pyros_msgs/opt_as_nested/tests/test_opt_int64.py index 5c82303..3f1f75e 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_int64.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_int64.py @@ -5,22 +5,15 @@ import os import sys import pytest +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) +import rosimport +rosimport.activate() -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +from . import msg as test_gen_msgs -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() - -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int8.py b/pyros_msgs/opt_as_nested/tests/test_opt_int8.py index 34169be..0637fe7 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_int8.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_int8.py @@ -6,19 +6,15 @@ import sys import pytest -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate - -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() - -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs + import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py b/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py index 55e356c..e99d866 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py @@ -2,26 +2,16 @@ import os import sys - - import pytest +import site -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate - -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs - import std_msgs.msg as std_msgs -except ImportError: # we should enter here if the message was not generated yet. - std_msgs, _ = msg_generate.generate_std_msgs() - pyros_msgs, _ = msg_generate.generate_pyros_msgs() +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) +import rosimport +rosimport.activate() +from . import msg as test_gen_msgs +import std_msgs.msg as std_msgs import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_string.py b/pyros_msgs/opt_as_nested/tests/test_opt_string.py index 36be2c2..a396975 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_string.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_string.py @@ -5,25 +5,19 @@ import os import sys import pytest +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +import rosimport +rosimport.activate() -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() +from . import msg as test_gen_msgs -try: - gen_test_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) import pyros_msgs.opt_as_nested # patching (need to know the field name) -pyros_msgs.opt_as_nested.duck_punch(gen_test_msgs.test_opt_string_as_nested, ['data']) +pyros_msgs.opt_as_nested.duck_punch(test_gen_msgs.test_opt_string_as_nested, ['data']) import hypothesis import hypothesis.strategies @@ -36,7 +30,7 @@ # hypothesis.strategies.binary() )) def test_init_rosdata(data): - msg = gen_test_msgs.test_opt_string_as_nested(data=data) + msg = test_gen_msgs.test_opt_string_as_nested(data=data) assert msg.data == data @@ -47,7 +41,7 @@ def test_init_rosdata(data): # hypothesis.strategies.binary() )) def test_init_data(data): - msg = gen_test_msgs.test_opt_string_as_nested(data=data) + msg = test_gen_msgs.test_opt_string_as_nested(data=data) assert msg.data == data @@ -58,12 +52,12 @@ def test_init_data(data): # hypothesis.strategies.binary() )) def test_init_raw(data): - msg = gen_test_msgs.test_opt_string_as_nested(data) + msg = test_gen_msgs.test_opt_string_as_nested(data) assert msg.data == data def test_init_default(): - msg = gen_test_msgs.test_opt_string_as_nested() + msg = test_gen_msgs.test_opt_string_as_nested() assert msg.data == None @@ -77,7 +71,7 @@ def test_init_default(): def test_wrong_init_except(data): """Testing we except when types do not match""" try: - gen_test_msgs.test_opt_string_as_nested(data) + test_gen_msgs.test_opt_string_as_nested(data) except AttributeError: pass except UnicodeEncodeError: diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_time.py b/pyros_msgs/opt_as_nested/tests/test_opt_time.py index 768cd11..7325122 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_time.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_time.py @@ -5,22 +5,15 @@ import genpy import pytest +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate - -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs - import std_msgs.msg as std_msgs -except ImportError: # we should enter here if the message was not generated yet. - std_msgs, _ = msg_generate.generate_std_msgs() - pyros_msgs, _ = msg_generate.generate_pyros_msgs() - -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) import pyros_msgs.opt_as_nested diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py b/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py index 1b8064b..d9631d9 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py @@ -5,20 +5,15 @@ import os import sys import pytest +import site -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py b/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py index 3daada2..f01d637 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py @@ -5,21 +5,15 @@ import os import sys import pytest +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +import rosimport +rosimport.activate() -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() +from . import msg as test_gen_msgs -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py b/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py index ec505ba..304c853 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py @@ -5,20 +5,15 @@ import os import sys import pytest +import site -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() +import rosimport +rosimport.activate() + +from . import msg as test_gen_msgs -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py b/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py index 3e15d86..db19d93 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py @@ -5,20 +5,14 @@ import os import sys import pytest +import site -# generating all and accessing the required message class. -from pyros_msgs.opt_as_nested.tests import msg_generate +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -try: - # This should succeed if the message class was already generated - import pyros_msgs.msg as pyros_msgs -except ImportError: # we should enter here if the message was not generated yet. - pyros_msgs = msg_generate.generate_pyros_msgs() +import rosimport +rosimport.activate() -try: - test_gen_msgs, gen_test_srvs = msg_generate.generate_test_msgs() -except Exception as e: - pytest.raises(e) +from . import msg as test_gen_msgs import pyros_msgs.opt_as_nested # patching (need to know the field name) diff --git a/pyros_msgs/typecheck/tests/msg_generate.py b/pyros_msgs/typecheck/tests/msg_generate.py index 9022f31..7a37821 100644 --- a/pyros_msgs/typecheck/tests/msg_generate.py +++ b/pyros_msgs/typecheck/tests/msg_generate.py @@ -8,30 +8,31 @@ We need to generate all and import only one time ( in case we have one instance of pytest running multiple tests ) """ -from pyros_msgs.importer.rosmsg_generator import generate_msgsrv_nspkg, import_msgsrv - # These depends on file structure and should no be in functions # dependencies for our generated messages -std_msgs_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps', 'std_msgs', 'msg') +rosdeps_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') # our own test messages we need to generate test_gen_msg_dir = os.path.join(os.path.dirname(__file__), 'msg') - -# TODO : replace this by a clever custom importer -def generate_std_msgs(): - flist = os.listdir(std_msgs_dir) - generated = generate_msgsrv_nspkg( - [os.path.join(std_msgs_dir, f) for f in flist], - package='std_msgs', - dependencies=['std_msgs'], - include_path=['std_msgs:{0}'.format(std_msgs_dir)], - ns_pkg=True - ) - test_gen_msgs, test_gen_srvs = import_msgsrv(*generated) - - return test_gen_msgs, test_gen_srvs +import rosimport +rosimport.activate() + + +# # TODO : replace this by a clever custom importer +# def generate_std_msgs(): +# flist = os.listdir(std_msgs_dir) +# generated = generate_msgsrv_nspkg( +# [os.path.join(std_msgs_dir, f) for f in flist], +# package='std_msgs', +# dependencies=['std_msgs'], +# include_path=['std_msgs:{0}'.format(std_msgs_dir)], +# ns_pkg=True +# ) +# test_gen_msgs, test_gen_srvs = import_msgsrv(*generated) +# +# return test_gen_msgs, test_gen_srvs diff --git a/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py b/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py index e67d4e5..7e8ef99 100644 --- a/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py +++ b/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py @@ -1,22 +1,20 @@ from __future__ import absolute_import, division, print_function -import numpy -import pytest -from StringIO import StringIO - import os import sys +import numpy +from six import BytesIO + +import site +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -# generating all and accessing the required message class. -from pyros_msgs.typecheck.tests import msg_generate +import rosimport +rosimport.activate() -try: - # This should succeed if the message class was already generated - import std_msgs.msg as std_msgs -except ImportError: # we should enter here if the message was not generated yet. - std_msgs, std_srvs = msg_generate.generate_std_msgs() +# This should succeed if the message class was already generated +import std_msgs.msg as std_msgs import genpy @@ -30,7 +28,7 @@ def test_typechecker_serialize_deserialize_float32_inverse(value): """""" # sending - buff = StringIO() + buff = BytesIO() value.serialize(buff) serialized = buff.getvalue() buff.close() @@ -51,7 +49,7 @@ def test_typechecker_serialize_deserialize_time_inverse(value): """""" # sending - buff = StringIO() + buff = BytesIO() value.serialize(buff) serialized = buff.getvalue() buff.close() diff --git a/pyros_msgs/typecheck/tests/test_ros_mappings.py b/pyros_msgs/typecheck/tests/test_ros_mappings.py index 2782aea..046c4b6 100644 --- a/pyros_msgs/typecheck/tests/test_ros_mappings.py +++ b/pyros_msgs/typecheck/tests/test_ros_mappings.py @@ -4,17 +4,17 @@ import sys import numpy import pytest -from StringIO import StringIO +from six import BytesIO +import site -# generating all and accessing the required message class. -from pyros_msgs.typecheck.tests import msg_generate +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -try: - # This should succeed if the message class was already generated - import std_msgs.msg as std_msgs -except ImportError: # we should enter here if the message was not generated yet. - std_msgs, std_srvs = msg_generate.generate_std_msgs() +import rosimport +rosimport.activate() + +# This should succeed if the message class was already generated +import std_msgs.msg as std_msgs import genpy @@ -116,7 +116,7 @@ def test_typechecker_serialize_deserialize_inverse(msg_type_and_ok_value): value = tc(msg_type_and_ok_value[1]) # sending - buff = StringIO() + buff = BytesIO() value.serialize(buff) serialized = buff.getvalue() buff.close() @@ -143,7 +143,7 @@ def test_typechecker_typechecker_prevent_broken_values(msg_type_and_bad_value): value = tc(msg_type_and_bad_value[1]) # sending - buff = StringIO() + buff = BytesIO() value.serialize(buff) serialized = buff.getvalue() buff.close() diff --git a/pyros_msgs/typecheck/tests/test_typechecker_arrays.py b/pyros_msgs/typecheck/tests/test_typechecker_arrays.py index 0dc9904..7ea99cc 100644 --- a/pyros_msgs/typecheck/tests/test_typechecker_arrays.py +++ b/pyros_msgs/typecheck/tests/test_typechecker_arrays.py @@ -62,7 +62,7 @@ def test_boolarray_typechecker_breaks_on_bad_number_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: boolarray_type_checker(value) - assert "is not accepted by Array of Accepter from " in excinfo.value.message + assert "is not accepted by Array of Accepter " in excinfo.value.message @given(proper_list_strategy_selector(intarray_type_checker)) @@ -87,7 +87,7 @@ def test_intarray_typechecker_breaks_on_bad_number_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: intarray_type_checker(value) - assert "is not accepted by Array of MinMax [-42..13835058055282163712] of Any of set" in excinfo.value.message + assert "is not accepted by Array of MinMax [-42..13835058055282163712] of Any of " in excinfo.value.message @given(proper_list_strategy_selector(floatarray_type_checker)) @@ -112,7 +112,7 @@ def test_floatarray_typechecker_breaks_on_bad_number_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: floatarray_type_checker(value) - assert "is not accepted by Array of MinMax [-42.0..1.38350580553e+19] of Accepter from " in excinfo.value.message + assert "is not accepted by Array of MinMax " in excinfo.value.message @given(proper_list_strategy_selector(stringarray_type_checker)) @@ -137,4 +137,4 @@ def test_stringarray_typechecker_breaks_on_bad_number_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: stringarray_type_checker(value) - assert "is not accepted by Array of Any of set" in excinfo.value.message + assert "is not accepted by Array of Any of " in excinfo.value.message diff --git a/pyros_msgs/typecheck/tests/test_typechecker_basic.py b/pyros_msgs/typecheck/tests/test_typechecker_basic.py index 427093c..9933f72 100644 --- a/pyros_msgs/typecheck/tests/test_typechecker_basic.py +++ b/pyros_msgs/typecheck/tests/test_typechecker_basic.py @@ -55,7 +55,7 @@ def test_breaks_on_bad_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: bool_type_checker(value) - assert "is not accepted by Accepter from " in excinfo.value.message + assert "is not accepted by Accepter " in excinfo.value.message @given(proper_basic_strategy_selector(integer_type_checker)) # where we learn that in python booleans are ints... @@ -76,7 +76,7 @@ def test_typechecker_with_any_accepter_breaks_on_bad_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: integer_type_checker(value) - assert "is not accepted by Any of set" in excinfo.value.message + assert "is not accepted by Any of " in excinfo.value.message @given(proper_basic_strategy_selector(integer_type_checker_min_max)) @@ -97,7 +97,7 @@ def test_typechecker_with_minmax_accepter_breaks_on_bad_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: integer_type_checker_min_max(value) - assert "is not accepted by MinMax [-42..13835058055282163712] of Any of set" in excinfo.value.message + assert "is not accepted by MinMax [-42..13835058055282163712] of Any of " in excinfo.value.message @given(proper_basic_strategy_selector(float_type_checker)) # where we learn that in python @@ -120,7 +120,7 @@ def test_typechecker_breaks_on_bad_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: float_type_checker(value) - assert "is not accepted by Accepter from " in excinfo.value.message + assert "is not accepted by Accepter " in excinfo.value.message @given(proper_basic_strategy_selector(float_type_checker_min_max)) @@ -141,7 +141,7 @@ def test_typechecker_with_minmax_accepter_breaks_on_bad_nonfloat_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: float_type_checker_min_max(value) - assert "is not accepted by MinMax [-42.0..1.38350580553e+19] of Accepter from " in excinfo.value.message + assert "is not accepted by MinMax " in excinfo.value.message # Separate test because of float arithemtics... @@ -154,7 +154,7 @@ def test_typechecker_with_minmax_accepter_breaks_on_bad_float_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: float_type_checker_min_max(value) - assert "is not accepted by MinMax [-42.0..1.38350580553e+19] of Accepter from " in excinfo.value.message + assert "is not accepted by MinMax " in excinfo.value.message @given(proper_basic_strategy_selector(string_type_checker)) @@ -175,7 +175,7 @@ def test_typechecker_breaks_on_bad_number_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: string_type_checker(value) - assert "is not accepted by Any of set" in excinfo.value.message + assert "is not accepted by Any of " in excinfo.value.message # Separate test for unicode fanciness @@ -187,4 +187,4 @@ def test_typechecker_breaks_on_bad_text_values(value): """ with pytest.raises(TypeCheckerException) as excinfo: string_type_checker(value) - assert "is not accepted by Any of set" in excinfo.value.message + assert "is not accepted by Any of " in excinfo.value.message From b4147e863201f8d60d4cda1a02d2bcb411ed3e57 Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 14 Jul 2017 18:42:22 +0900 Subject: [PATCH 19/28] tox tests now passing on python 2.7 with dev version of rosimport. --- .gitmodules | 3 + dev-requirements.txt | 3 +- pydeps/rosimport | 1 + pyros_msgs/opt_as_array/tests/__init__.py | 11 +++ pyros_msgs/opt_as_array/tests/msg_generate.py | 56 ------------- .../opt_as_array/tests/test_opt_bool.py | 9 -- .../opt_as_array/tests/test_opt_duration.py | 10 --- .../opt_as_array/tests/test_opt_int16.py | 11 --- .../opt_as_array/tests/test_opt_int32.py | 9 -- .../opt_as_array/tests/test_opt_int64.py | 9 -- .../opt_as_array/tests/test_opt_int8.py | 8 -- .../opt_as_array/tests/test_opt_std_empty.py | 10 --- .../opt_as_array/tests/test_opt_string.py | 10 --- .../opt_as_array/tests/test_opt_time.py | 10 --- .../opt_as_array/tests/test_opt_uint16.py | 8 -- .../opt_as_array/tests/test_opt_uint32.py | 8 -- .../opt_as_array/tests/test_opt_uint64.py | 8 -- .../opt_as_array/tests/test_opt_uint8.py | 8 -- pyros_msgs/opt_as_nested/tests/__init__.py | 11 +++ .../opt_as_nested/tests/msg_generate.py | 82 ------------------- .../opt_as_nested/tests/test_opt_bool.py | 8 -- .../opt_as_nested/tests/test_opt_duration.py | 10 --- .../opt_as_nested/tests/test_opt_int16.py | 10 --- .../opt_as_nested/tests/test_opt_int32.py | 9 -- .../opt_as_nested/tests/test_opt_int64.py | 10 --- .../opt_as_nested/tests/test_opt_int8.py | 10 --- .../opt_as_nested/tests/test_opt_std_empty.py | 8 -- .../opt_as_nested/tests/test_opt_string.py | 10 --- .../opt_as_nested/tests/test_opt_time.py | 9 -- .../opt_as_nested/tests/test_opt_uint16.py | 10 --- .../opt_as_nested/tests/test_opt_uint32.py | 10 --- .../opt_as_nested/tests/test_opt_uint64.py | 9 -- .../opt_as_nested/tests/test_opt_uint8.py | 9 -- pyros_msgs/typecheck/tests/msg_generate.py | 38 --------- .../tests/test_basic_ros_serialization.py | 2 +- .../typecheck/tests/test_ros_mappings.py | 2 +- .../tests/test_typechecker_arrays.py | 10 --- .../typecheck/tests/test_typechecker_basic.py | 9 -- .../tests/test_typechecker_nested.py | 11 --- ros-site/rosimport.pth | 1 + setup.py | 45 +--------- tox.ini | 11 +-- 42 files changed, 38 insertions(+), 498 deletions(-) create mode 160000 pydeps/rosimport delete mode 100644 pyros_msgs/opt_as_array/tests/msg_generate.py delete mode 100644 pyros_msgs/opt_as_nested/tests/msg_generate.py delete mode 100644 pyros_msgs/typecheck/tests/msg_generate.py create mode 100644 ros-site/rosimport.pth diff --git a/.gitmodules b/.gitmodules index 2351cfc..aaf315d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "rosdeps/ros_comm_msgs"] path = rosdeps/ros_comm_msgs url = https://github.com/ros/ros_comm_msgs.git +[submodule "pydeps/rosimport"] + path = pydeps/rosimport + url = https://github.com/asmodehn/rosimport diff --git a/dev-requirements.txt b/dev-requirements.txt index ebc207a..3eb68bf 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,4 +4,5 @@ twine # source access to latest filefinder from git ... -e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 --e git+https://github.com/asmodehn/rosimport.git#egg=rosimport +# currently satisfied by submodule +#-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport diff --git a/pydeps/rosimport b/pydeps/rosimport new file mode 160000 index 0000000..e806ef2 --- /dev/null +++ b/pydeps/rosimport @@ -0,0 +1 @@ +Subproject commit e806ef2a05279a6d0b86b0ffa577a92048272c5c diff --git a/pyros_msgs/opt_as_array/tests/__init__.py b/pyros_msgs/opt_as_array/tests/__init__.py index e69de29..3355f20 100644 --- a/pyros_msgs/opt_as_array/tests/__init__.py +++ b/pyros_msgs/opt_as_array/tests/__init__.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import, division, print_function + +import os +import sys +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'pydeps')) + +import rosimport +rosimport.activate() diff --git a/pyros_msgs/opt_as_array/tests/msg_generate.py b/pyros_msgs/opt_as_array/tests/msg_generate.py deleted file mode 100644 index ac556aa..0000000 --- a/pyros_msgs/opt_as_array/tests/msg_generate.py +++ /dev/null @@ -1,56 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import os -import sys - -""" -module handling test message generation and import. -We need to generate all and import only one time ( in case we have one instance of pytest running multiple tests ) -""" - -from pyros_msgs.importer.rosmsg_generator import generate_msgsrv_nspkg, import_msgsrv - -# These depends on file structure and should no be in functions - -# dependencies for our generated messages -std_msgs_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps', 'std_msgs', 'msg') - -# our own test messages we need to generate -test_gen_msg_dir = os.path.join(os.path.dirname(__file__), 'msg') - - -# TODO : replace this by a clever custom importer -def generate_std_msgs(): - flist = os.listdir(std_msgs_dir) - generated = generate_msgsrv_nspkg( - [os.path.join(std_msgs_dir, f) for f in flist], - package='std_msgs', - dependencies=['std_msgs'], - include_path=['std_msgs:{0}'.format(std_msgs_dir)], - ns_pkg=True - ) - std_msgs, std_srvs = import_msgsrv(*generated) - - return std_msgs, std_srvs - - -def generate_test_msgs(): - try: - # This should succeed if the message has been generated previously. - import std_msgs.msg as std_msgs - except ImportError: # we should enter here if the message class hasnt been generated yet. - std_msgs, std_srvs = generate_std_msgs() - - flist = os.listdir(test_gen_msg_dir) - generated = generate_msgsrv_nspkg( - [os.path.join(test_gen_msg_dir, f) for f in flist], - package='test_array_gen_msgs', - dependencies=['std_msgs'], - include_path=['std_msgs:{0}'.format(std_msgs_dir)], - ns_pkg=True - ) - test_gen_msgs, test_gen_srvs = import_msgsrv(*generated) - - return test_gen_msgs, test_gen_srvs - - diff --git a/pyros_msgs/opt_as_array/tests/test_opt_bool.py b/pyros_msgs/opt_as_array/tests/test_opt_bool.py index 706650f..bc2bf74 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_bool.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_bool.py @@ -1,14 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import sys -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_duration.py b/pyros_msgs/opt_as_array/tests/test_opt_duration.py index 252bdf4..ec233af 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_duration.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_duration.py @@ -1,16 +1,6 @@ from __future__ import absolute_import, division, print_function -import os -import sys - - import genpy -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int16.py b/pyros_msgs/opt_as_array/tests/test_opt_int16.py index 9906c46..ae46014 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_int16.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_int16.py @@ -1,18 +1,7 @@ from __future__ import absolute_import, division, print_function -import os -import sys - -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs - import pyros_msgs.opt_as_array # patching (need to know the field name) pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_int16_as_array, ['data']) diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int32.py b/pyros_msgs/opt_as_array/tests/test_opt_int32.py index 78cc4b3..3b39c25 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_int32.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_int32.py @@ -1,14 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import sys - -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int64.py b/pyros_msgs/opt_as_array/tests/test_opt_int64.py index 7d4c21f..5deef74 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_int64.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_int64.py @@ -1,14 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import sys -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int8.py b/pyros_msgs/opt_as_array/tests/test_opt_int8.py index 3226707..dcc9843 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_int8.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_int8.py @@ -1,13 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py b/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py index f699821..a8e7adf 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py @@ -1,16 +1,6 @@ from __future__ import absolute_import, division, print_function -import os -import sys - - import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs import std_msgs.msg as std_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_string.py b/pyros_msgs/opt_as_array/tests/test_opt_string.py index e4f9135..06bd1bb 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_string.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_string.py @@ -1,15 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import sys - -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_time.py b/pyros_msgs/opt_as_array/tests/test_opt_time.py index 018d39f..02190db 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_time.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_time.py @@ -1,20 +1,10 @@ from __future__ import absolute_import, division, print_function -import os -import sys - import genpy -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs - import pyros_msgs.opt_as_array # patching (need to know the field name) pyros_msgs.opt_as_array.duck_punch(test_gen_msgs.test_opt_time_as_array, ['data']) diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint16.py b/pyros_msgs/opt_as_array/tests/test_opt_uint16.py index 7f59679..0e8000b 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_uint16.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_uint16.py @@ -1,13 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint32.py b/pyros_msgs/opt_as_array/tests/test_opt_uint32.py index 8be00e8..151f7df 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_uint32.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_uint32.py @@ -1,13 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs import pyros_msgs.opt_as_array diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint64.py b/pyros_msgs/opt_as_array/tests/test_opt_uint64.py index 7277854..0624c1e 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_uint64.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_uint64.py @@ -1,13 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint8.py b/pyros_msgs/opt_as_array/tests/test_opt_uint8.py index df9daad..3c46911 100644 --- a/pyros_msgs/opt_as_array/tests/test_opt_uint8.py +++ b/pyros_msgs/opt_as_array/tests/test_opt_uint8.py @@ -1,13 +1,5 @@ from __future__ import absolute_import, division, print_function -import os -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/__init__.py b/pyros_msgs/opt_as_nested/tests/__init__.py index e69de29..8856266 100644 --- a/pyros_msgs/opt_as_nested/tests/__init__.py +++ b/pyros_msgs/opt_as_nested/tests/__init__.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import, division, print_function + +import os +import sys +import site + +# This is used for message definitions, not for python code +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) + +import rosimport +rosimport.activate() diff --git a/pyros_msgs/opt_as_nested/tests/msg_generate.py b/pyros_msgs/opt_as_nested/tests/msg_generate.py deleted file mode 100644 index 3b2dbd7..0000000 --- a/pyros_msgs/opt_as_nested/tests/msg_generate.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import os -import sys - -""" -module handling test message generation and import. -We need to generate all and import only one time ( in case we have one instance of pytest running multiple tests ) -""" - -# These depends on file structure and should no be in functions - -# dependencies for our generated messages -pyros_msgs_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) -rosdeps_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') - - -# our own test messages we need to generate -test_gen_msg_dir = os.path.join(os.path.dirname(__file__), 'msg') - - -import rosimport -rosimport.activate_hook_for(rosdeps_dir) - -# # TODO : replace this by a clever custom importer -# def generate_pyros_msgs(): -# flist = os.listdir(pyros_msgs_dir) -# #flist = flist + os.listdir(std_msgs_dir) -# generated = generate_msgsrv_nspkg( -# [os.path.join(pyros_msgs_dir, f) for f in flist], -# package='pyros_msgs', -# dependencies=['pyros_msgs'], -# include_path=['pyros_msgs:{0}'.format(pyros_msgs_dir)], -# ns_pkg=True -# ) -# pyros_msgs_msg, pyros_srvs_srv = import_msgsrv(*generated) -# -# return pyros_msgs_msg, pyros_srvs_srv -# -# def generate_std_msgs(): -# flist = os.listdir(std_msgs_dir) -# generated = generate_msgsrv_nspkg( -# [os.path.join(std_msgs_dir, f) for f in flist], -# package='std_msgs', -# dependencies=['std_msgs'], -# include_path=['std_msgs:{0}'.format(std_msgs_dir)], -# ns_pkg=True -# ) -# std_msgs, std_srvs = import_msgsrv(*generated) -# -# return std_msgs, std_srvs -# -# -# def generate_test_msgs(): -# try: -# # This should succeed if the message has been generated previously (or accessing ROS generated message) -# import pyros_msgs.msg as pyros_msgs -# except ImportError: # we should enter here if the message class hasn't been generated yet. -# pyros_msgs_msg, pyros_msgs_srv = generate_pyros_msgs() -# -# try: -# # This should succeed if the message class was already generated -# import std_msgs.msg as std_msgs -# except ImportError: # we should enter here if the message was not generated yet. -# _, _ = generate_std_msgs() -# -# flist = os.listdir(test_gen_msg_dir) -# #flist = flist + os.listdir(std_msgs_dir) -# generated = generate_msgsrv_nspkg( -# [os.path.join(test_gen_msg_dir, f) for f in flist], -# package='test_nested_gen_msgs', -# dependencies=['pyros_msgs', 'std_msgs'], -# include_path=['pyros_msgs:{0}'.format(pyros_msgs_dir), 'std_msgs:{0}'.format(std_msgs_dir)], -# ns_pkg=True -# ) -# test_gen_msgs, test_gen_srvs = import_msgsrv(*generated) -# -# return test_gen_msgs, test_gen_srvs - - - - diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_bool.py b/pyros_msgs/opt_as_nested/tests/test_opt_bool.py index 6ba2caf..7a5a204 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_bool.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_bool.py @@ -2,15 +2,7 @@ # TODO : check all types -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_duration.py b/pyros_msgs/opt_as_nested/tests/test_opt_duration.py index d10454d..6558df8 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_duration.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_duration.py @@ -1,19 +1,9 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import genpy import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int16.py b/pyros_msgs/opt_as_nested/tests/test_opt_int16.py index 4a10a55..2b6cff2 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_int16.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_int16.py @@ -1,17 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int32.py b/pyros_msgs/opt_as_nested/tests/test_opt_int32.py index 75d584e..2523870 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_int32.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_int32.py @@ -1,16 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int64.py b/pyros_msgs/opt_as_nested/tests/test_opt_int64.py index 3f1f75e..0a1b734 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_int64.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_int64.py @@ -1,17 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int8.py b/pyros_msgs/opt_as_nested/tests/test_opt_int8.py index 0637fe7..f25daca 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_int8.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_int8.py @@ -1,18 +1,8 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py b/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py index e99d866..3469203 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py @@ -1,14 +1,6 @@ from __future__ import absolute_import, division, print_function -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs import std_msgs.msg as std_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_string.py b/pyros_msgs/opt_as_nested/tests/test_opt_string.py index a396975..e6c7369 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_string.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_string.py @@ -1,17 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_time.py b/pyros_msgs/opt_as_nested/tests/test_opt_time.py index 7325122..2bc7cd8 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_time.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_time.py @@ -1,16 +1,7 @@ from __future__ import absolute_import, division, print_function -import os -import sys - import genpy import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py b/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py index d9631d9..baa701e 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py @@ -1,17 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py b/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py index f01d637..6f31884 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py @@ -1,17 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() - from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py b/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py index 304c853..88f5e2b 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py @@ -1,16 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py b/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py index db19d93..79ad93f 100644 --- a/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py +++ b/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py @@ -1,16 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types - -import os -import sys import pytest -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() from . import msg as test_gen_msgs diff --git a/pyros_msgs/typecheck/tests/msg_generate.py b/pyros_msgs/typecheck/tests/msg_generate.py deleted file mode 100644 index 7a37821..0000000 --- a/pyros_msgs/typecheck/tests/msg_generate.py +++ /dev/null @@ -1,38 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import os -import sys - -""" -module handling test message generation and import. -We need to generate all and import only one time ( in case we have one instance of pytest running multiple tests ) -""" - -# These depends on file structure and should no be in functions - -# dependencies for our generated messages -rosdeps_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps') - -# our own test messages we need to generate -test_gen_msg_dir = os.path.join(os.path.dirname(__file__), 'msg') - -import rosimport -rosimport.activate() - - -# # TODO : replace this by a clever custom importer -# def generate_std_msgs(): -# flist = os.listdir(std_msgs_dir) -# generated = generate_msgsrv_nspkg( -# [os.path.join(std_msgs_dir, f) for f in flist], -# package='std_msgs', -# dependencies=['std_msgs'], -# include_path=['std_msgs:{0}'.format(std_msgs_dir)], -# ns_pkg=True -# ) -# test_gen_msgs, test_gen_srvs = import_msgsrv(*generated) -# -# return test_gen_msgs, test_gen_srvs - - - diff --git a/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py b/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py index 7e8ef99..08a5b90 100644 --- a/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py +++ b/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py @@ -7,7 +7,7 @@ from six import BytesIO import site - +# This is used for message definitions, not for python code site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) import rosimport diff --git a/pyros_msgs/typecheck/tests/test_ros_mappings.py b/pyros_msgs/typecheck/tests/test_ros_mappings.py index 046c4b6..784cce7 100644 --- a/pyros_msgs/typecheck/tests/test_ros_mappings.py +++ b/pyros_msgs/typecheck/tests/test_ros_mappings.py @@ -7,7 +7,7 @@ from six import BytesIO import site - +# This is used for message definitions, not for python code site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) import rosimport diff --git a/pyros_msgs/typecheck/tests/test_typechecker_arrays.py b/pyros_msgs/typecheck/tests/test_typechecker_arrays.py index 7ea99cc..911719e 100644 --- a/pyros_msgs/typecheck/tests/test_typechecker_arrays.py +++ b/pyros_msgs/typecheck/tests/test_typechecker_arrays.py @@ -2,16 +2,6 @@ import pytest import sys - -# try: -# import pyros_msgs -# import genpy -# except ImportError: -# import pyros_setup -# pyros_setup.configurable_import().configure().activate() -# import pyros_msgs -# import genpy - import six from pyros_msgs.typecheck.typechecker import ( diff --git a/pyros_msgs/typecheck/tests/test_typechecker_basic.py b/pyros_msgs/typecheck/tests/test_typechecker_basic.py index 9933f72..6ad24af 100644 --- a/pyros_msgs/typecheck/tests/test_typechecker_basic.py +++ b/pyros_msgs/typecheck/tests/test_typechecker_basic.py @@ -1,14 +1,5 @@ from __future__ import absolute_import, division, print_function, unicode_literals -# try: -# import pyros_msgs -# import genpy -# except ImportError: -# import pyros_setup -# pyros_setup.configurable_import().configure().activate() -# import pyros_msgs -# import genpy - import six from pyros_msgs.typecheck.typechecker import ( diff --git a/pyros_msgs/typecheck/tests/test_typechecker_nested.py b/pyros_msgs/typecheck/tests/test_typechecker_nested.py index ba9a243..115eebc 100644 --- a/pyros_msgs/typecheck/tests/test_typechecker_nested.py +++ b/pyros_msgs/typecheck/tests/test_typechecker_nested.py @@ -1,16 +1,5 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import sys - -# try: -# import pyros_msgs -# import genpy -# except ImportError: -# import pyros_setup -# pyros_setup.configurable_import().configure().activate() -# import pyros_msgs -# import genpy - import six from pyros_msgs.typecheck.typechecker import ( diff --git a/ros-site/rosimport.pth b/ros-site/rosimport.pth new file mode 100644 index 0000000..dfc2d75 --- /dev/null +++ b/ros-site/rosimport.pth @@ -0,0 +1 @@ +../pydeps/rosimport \ No newline at end of file diff --git a/setup.py b/setup.py index 53439a5..a986c1c 100644 --- a/setup.py +++ b/setup.py @@ -12,11 +12,6 @@ version = runpy.run_path('pyros_msgs/typecheck/_version.py') __version__ = version.get('__version__') -# Including generator module directly from code to be able to generate our message classes -# Ref : http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path -import imp -rosmsg_generator = imp.load_source('rosmsg_generator', 'pyros_msgs/importer/rosmsg_generator.py') - # Best Flow : # Clean previous build & dist @@ -30,40 +25,6 @@ # TODO : command to retrieve extra ROS stuff from a third party release repo ( for ROS devs ). useful in dev only so maybe "rosdevelop" ? or via catkin_pip ? # TODO : command to release to Pip and ROS (bloom) same version one after the other... - -# Clean way to add a custom "python setup.py " -# Ref setup.py command extension : https://blog.niteoweb.com/setuptools-run-custom-code-in-setup-py/ -class GenerateMsgCommand(setuptools.Command): - """Command to generate message class""" - description = "generate messages for pyros_msgs" - user_options = [] - - def initialize_options(self): - """init options""" - # TODO : pass distro path [indigo|jade|etc.] - pass - - def finalize_options(self): - """finalize options""" - pass - - def run(self): - """runner""" - - # generating message class - generated = rosmsg_generator.generate_msgsrv_nspkg( - [os.path.join(os.path.dirname(__file__), 'msg', 'OptionalFields.msg')], - package='pyros_msgs', - ns_pkg=False, # no need to generate ns_pkg here, we can use the one we already have - ) - - # Note we have a tricky problem here since the ros distro for our target needs to be installed on the machine packaging this... - # But pip packages are supposed to work on any platform, so we might need another way... - - print("Check that the messages classes have been generated properly...") - sys.exit() - - # Clean way to add a custom "python setup.py " # Ref setup.py command extension : https://blog.niteoweb.com/setuptools-run-custom-code-in-setup-py/ class PrepareReleaseCommand(setuptools.Command): @@ -235,7 +196,7 @@ def run(self): license='MIT', packages=[ 'pyros_msgs', - # 'pyros_msgs.msg', #TODO : generate this for pure python package, in a way that is compatible with catkin (so we can still use catkin_make with this) + # 'pyros_msgs.msg', #catkin build should generate it anyway. usual python build can rely on rosimport. 'pyros_msgs.typecheck', 'pyros_msgs.typecheck.tests', 'pyros_msgs.opt_as_array', 'pyros_msgs.opt_as_array.tests', 'pyros_msgs.opt_as_nested', 'pyros_msgs.opt_as_nested.tests', @@ -245,11 +206,14 @@ def run(self): include_package_data=True, # use MANIFEST.in during install. # Reference for optional dependencies : http://stackoverflow.com/questions/4796936/does-pip-handle-extras-requires-from-setuptools-distribute-based-sources install_requires=[ + 'six', # this is needed as install dependency since we embed tests in the package. # 'pyros_setup>=0.2.1', # needed to grab ros environment even if distro setup.sh not sourced # 'pyros_utils', # this must be satisfied by the ROS package system... # 'importlib2>=3.4;python_version<"3.4"', # NOT working we use a patched version of it, through a symlink (to make linux deb release possible) 'filefinder2; python_version<"3.4"', # we rely on this for PEP420 on python 2.7 + # TMP : not released yet + # 'rosimport', # we rely on this for generating ROS message if necessary before importing 'pyyaml>=3.10', # genpy relies on this... 'pytest>=2.8.0', # as per hypothesis requirement (careful with 2.5.1 on trusty) 'pytest-xdist', # for --boxed (careful with the version it will be moved out of xdist) @@ -257,7 +221,6 @@ def run(self): 'numpy>=1.8.2', # from trusty version ], cmdclass={ - 'generatemsg': GenerateMsgCommand, 'rosdevelop': RosDevelopCommand, 'rospublish': ROSPublishCommand, }, diff --git a/tox.ini b/tox.ini index 8e93923..8767d59 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ skipsdist=True [testenv] deps = # TODO : check why / how install_requires are installed or not in tox environments... - pytest + -rdev-requirements.txt # to always force recreation and avoid unexpected side effects recreate=True @@ -39,6 +39,8 @@ recreate=True # sitepackages=True # We do not want any access to system (same as basic travis python testing) +# TMP because our dependencies are not in pips +usedevelop = True # we need to set our ROS distro in the pythonpath for our code to rely on ROS #TODO : not havnig this breaks pyros-setup which tries to create its config file in @@ -58,15 +60,8 @@ setenv= kinetic: PYTHONPATH=/opt/ros/kinetic/lib/python2.7/dist-packages commands= - # First we need to generate our messages (just like ROS catkin build flow would) - python setup.py generatemsg - python setup.py develop - # we use the develop way documented in http://tox.readthedocs.io/en/latest/example/general.html, - # to be able to use generated messages from setup.py, as well as ros-site. - # we want to make sure python finds the installed package in tox env # and doesn't confuse with pyc generated during dev (which happens if we use self test feature here) - py.test --pyargs pyros_msgs/importer/tests {posargs} --boxed py.test --pyargs pyros_msgs/typecheck/tests {posargs} py.test --pyargs pyros_msgs/opt_as_array/tests {posargs} py.test --pyargs pyros_msgs/opt_as_nested/tests {posargs} From 9184e7b1b312cc7ac97ed16f85009fb8ade93067 Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 18 Jul 2017 15:55:03 +0900 Subject: [PATCH 20/28] moving tests out of package to avoid changing python import logic without the final user knowing. --- dev-requirements.txt | 5 ++--- msg/OptionalFields.msg | 1 + pyros_msgs/opt_as_array/tests/__init__.py | 11 ---------- pyros_msgs/opt_as_nested/tests/__init__.py | 11 ---------- pyros_msgs/typecheck/ros_mappings.py | 11 ++++------ setup.py | 13 ++++++------ tests/__init__.py | 0 tests/opt_as_array/__init__.py | 21 +++++++++++++++++++ .../msg/test_opt_bool_as_array.msg | 0 .../msg/test_opt_duration_as_array.msg | 0 .../msg/test_opt_float32_as_array.msg | 0 .../msg/test_opt_float64_as_array.msg | 0 .../msg/test_opt_header_as_array.msg | 0 .../msg/test_opt_int16_as_array.msg | 0 .../msg/test_opt_int32_as_array.msg | 0 .../msg/test_opt_int64_as_array.msg | 0 .../msg/test_opt_int8_as_array.msg | 0 .../msg/test_opt_std_empty_as_array.msg | 0 .../msg/test_opt_string_as_array.msg | 0 .../msg/test_opt_time_as_array.msg | 0 .../msg/test_opt_uint16_as_array.msg | 0 .../msg/test_opt_uint32_as_array.msg | 0 .../msg/test_opt_uint64_as_array.msg | 0 .../msg/test_opt_uint8_as_array.msg | 0 .../opt_as_array}/test_opt_bool.py | 0 .../opt_as_array}/test_opt_duration.py | 0 .../opt_as_array}/test_opt_int16.py | 0 .../opt_as_array}/test_opt_int32.py | 0 .../opt_as_array}/test_opt_int64.py | 0 .../opt_as_array}/test_opt_int8.py | 0 .../opt_as_array}/test_opt_std_empty.py | 0 .../opt_as_array}/test_opt_string.py | 0 .../opt_as_array}/test_opt_time.py | 0 .../opt_as_array}/test_opt_uint16.py | 0 .../opt_as_array}/test_opt_uint32.py | 0 .../opt_as_array}/test_opt_uint64.py | 0 .../opt_as_array}/test_opt_uint8.py | 0 tests/opt_as_nested/__init__.py | 21 +++++++++++++++++++ .../msg/test_opt_bool_as_nested.msg | 0 .../msg/test_opt_duration_as_nested.msg | 0 .../msg/test_opt_int16_as_nested.msg | 0 .../msg/test_opt_int32_as_nested.msg | 0 .../msg/test_opt_int64_as_nested.msg | 0 .../msg/test_opt_int8_as_nested.msg | 0 .../msg/test_opt_std_empty_as_nested.msg | 0 .../msg/test_opt_string_as_nested.msg | 0 .../msg/test_opt_time_as_nested.msg | 0 .../msg/test_opt_uint16_as_nested.msg | 0 .../msg/test_opt_uint32_as_nested.msg | 0 .../msg/test_opt_uint64_as_nested.msg | 0 .../msg/test_opt_uint8_as_nested.msg | 0 .../opt_as_nested}/test_opt_bool.py | 0 .../opt_as_nested}/test_opt_duration.py | 0 .../opt_as_nested}/test_opt_int16.py | 0 .../opt_as_nested}/test_opt_int32.py | 0 .../opt_as_nested}/test_opt_int64.py | 0 .../opt_as_nested}/test_opt_int8.py | 0 .../opt_as_nested}/test_opt_std_empty.py | 0 .../opt_as_nested}/test_opt_string.py | 0 .../opt_as_nested}/test_opt_time.py | 0 .../opt_as_nested}/test_opt_uint16.py | 0 .../opt_as_nested}/test_opt_uint32.py | 0 .../opt_as_nested}/test_opt_uint64.py | 0 .../opt_as_nested}/test_opt_uint8.py | 0 .../tests => tests/typecheck}/__init__.py | 0 .../test_basic_ros_serialization.py | 2 +- .../typecheck}/test_ros_mappings.py | 2 +- .../typecheck}/test_typechecker_arrays.py | 0 .../typecheck}/test_typechecker_basic.py | 0 .../typecheck}/test_typechecker_nested.py | 0 tox.ini | 4 +--- 71 files changed, 59 insertions(+), 43 deletions(-) delete mode 100644 pyros_msgs/opt_as_array/tests/__init__.py delete mode 100644 pyros_msgs/opt_as_nested/tests/__init__.py create mode 100644 tests/__init__.py create mode 100644 tests/opt_as_array/__init__.py rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_bool_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_duration_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_float32_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_float64_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_header_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_int16_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_int32_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_int64_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_int8_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_std_empty_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_string_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_time_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_uint16_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_uint32_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_uint64_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/msg/test_opt_uint8_as_array.msg (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_bool.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_duration.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_int16.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_int32.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_int64.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_int8.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_std_empty.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_string.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_time.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_uint16.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_uint32.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_uint64.py (100%) rename {pyros_msgs/opt_as_array/tests => tests/opt_as_array}/test_opt_uint8.py (100%) create mode 100644 tests/opt_as_nested/__init__.py rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_bool_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_duration_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_int16_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_int32_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_int64_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_int8_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_std_empty_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_string_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_time_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_uint16_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_uint32_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_uint64_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/msg/test_opt_uint8_as_nested.msg (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_bool.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_duration.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_int16.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_int32.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_int64.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_int8.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_std_empty.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_string.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_time.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_uint16.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_uint32.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_uint64.py (100%) rename {pyros_msgs/opt_as_nested/tests => tests/opt_as_nested}/test_opt_uint8.py (100%) rename {pyros_msgs/typecheck/tests => tests/typecheck}/__init__.py (100%) rename {pyros_msgs/typecheck/tests => tests/typecheck}/test_basic_ros_serialization.py (96%) rename {pyros_msgs/typecheck/tests => tests/typecheck}/test_ros_mappings.py (99%) rename {pyros_msgs/typecheck/tests => tests/typecheck}/test_typechecker_arrays.py (100%) rename {pyros_msgs/typecheck/tests => tests/typecheck}/test_typechecker_basic.py (100%) rename {pyros_msgs/typecheck/tests => tests/typecheck}/test_typechecker_nested.py (100%) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3eb68bf..093784a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,7 @@ -#pathlib2 gitchangelog twine +rosimport # source access to latest filefinder from git ... --e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 -# currently satisfied by submodule +#-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 #-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport diff --git a/msg/OptionalFields.msg b/msg/OptionalFields.msg index f0a219a..3f5c1c8 100644 --- a/msg/OptionalFields.msg +++ b/msg/OptionalFields.msg @@ -1,5 +1,6 @@ # The list of fields names that should be considered optional string[] optional_field_names +# TODO: find a way to make that static, in the definition itself, and not dynamically... # A boolean value per field, indicating if that field has been initialized or not bool[] optional_field_initialized_ \ No newline at end of file diff --git a/pyros_msgs/opt_as_array/tests/__init__.py b/pyros_msgs/opt_as_array/tests/__init__.py deleted file mode 100644 index 3355f20..0000000 --- a/pyros_msgs/opt_as_array/tests/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import os -import sys -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'pydeps')) - -import rosimport -rosimport.activate() diff --git a/pyros_msgs/opt_as_nested/tests/__init__.py b/pyros_msgs/opt_as_nested/tests/__init__.py deleted file mode 100644 index 8856266..0000000 --- a/pyros_msgs/opt_as_nested/tests/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import os -import sys -import site - -# This is used for message definitions, not for python code -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) - -import rosimport -rosimport.activate() diff --git a/pyros_msgs/typecheck/ros_mappings.py b/pyros_msgs/typecheck/ros_mappings.py index cceb128..0492693 100644 --- a/pyros_msgs/typecheck/ros_mappings.py +++ b/pyros_msgs/typecheck/ros_mappings.py @@ -9,20 +9,17 @@ import os +# TMP : waiting for proper pip installable version of genpy and genmsd try: # Using genpy directly if ROS has been setup (while using from ROS pkg) - import genpy - + import genpy, genmsg except ImportError: - # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) - import site site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'ros-site')) + import genpy, genmsg - import genpy - - # Note we do not want to use pyros_setup here. + # Note we do not want to use pyros_setup/rosimport here. # We do not want to do a full ROS setup, only import specific packages. # If needed it should have been done before (loading a parent package). # this handle the case where we want to be independent of any underlying ROS system. diff --git a/setup.py b/setup.py index a986c1c..8120564 100644 --- a/setup.py +++ b/setup.py @@ -196,10 +196,10 @@ def run(self): license='MIT', packages=[ 'pyros_msgs', - # 'pyros_msgs.msg', #catkin build should generate it anyway. usual python build can rely on rosimport. - 'pyros_msgs.typecheck', 'pyros_msgs.typecheck.tests', - 'pyros_msgs.opt_as_array', 'pyros_msgs.opt_as_array.tests', - 'pyros_msgs.opt_as_nested', 'pyros_msgs.opt_as_nested.tests', + # 'pyros_msgs.msg', #catkin build should generate it anyway. python build can assume client uses rosimport. + 'pyros_msgs.typecheck', + 'pyros_msgs.opt_as_array', + 'pyros_msgs.opt_as_nested', ], namespace_packages=['pyros_msgs'], # this is better than using package data ( since behavior is a bit different from distutils... ) @@ -212,14 +212,15 @@ def run(self): # 'pyros_utils', # this must be satisfied by the ROS package system... # 'importlib2>=3.4;python_version<"3.4"', # NOT working we use a patched version of it, through a symlink (to make linux deb release possible) 'filefinder2; python_version<"3.4"', # we rely on this for PEP420 on python 2.7 - # TMP : not released yet - # 'rosimport', # we rely on this for generating ROS message if necessary before importing 'pyyaml>=3.10', # genpy relies on this... 'pytest>=2.8.0', # as per hypothesis requirement (careful with 2.5.1 on trusty) 'pytest-xdist', # for --boxed (careful with the version it will be moved out of xdist) 'hypothesis>=3.0.1', # to target xenial LTS version 'numpy>=1.8.2', # from trusty version ], + test_requires=[ + 'rosimport', # we rely on this for generating ROS message if necessary before importing + ], cmdclass={ 'rosdevelop': RosDevelopCommand, 'rospublish': ROSPublishCommand, diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/opt_as_array/__init__.py b/tests/opt_as_array/__init__.py new file mode 100644 index 0000000..d544393 --- /dev/null +++ b/tests/opt_as_array/__init__.py @@ -0,0 +1,21 @@ +from __future__ import absolute_import, division, print_function + +import os +import sys +import site + +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps')) +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'pydeps')) + +# TMP : waiting for proper pip installable version of genpy and genmsd +try: + # Using genpy directly if ROS has been setup (while using from ROS pkg) + import genpy, genmsg +except ImportError: + # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) + import site + site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'ros-site')) + import genpy, genmsg + +import rosimport +rosimport.activate() diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_bool_as_array.msg b/tests/opt_as_array/msg/test_opt_bool_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_bool_as_array.msg rename to tests/opt_as_array/msg/test_opt_bool_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_duration_as_array.msg b/tests/opt_as_array/msg/test_opt_duration_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_duration_as_array.msg rename to tests/opt_as_array/msg/test_opt_duration_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_float32_as_array.msg b/tests/opt_as_array/msg/test_opt_float32_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_float32_as_array.msg rename to tests/opt_as_array/msg/test_opt_float32_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_float64_as_array.msg b/tests/opt_as_array/msg/test_opt_float64_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_float64_as_array.msg rename to tests/opt_as_array/msg/test_opt_float64_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_header_as_array.msg b/tests/opt_as_array/msg/test_opt_header_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_header_as_array.msg rename to tests/opt_as_array/msg/test_opt_header_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_int16_as_array.msg b/tests/opt_as_array/msg/test_opt_int16_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_int16_as_array.msg rename to tests/opt_as_array/msg/test_opt_int16_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_int32_as_array.msg b/tests/opt_as_array/msg/test_opt_int32_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_int32_as_array.msg rename to tests/opt_as_array/msg/test_opt_int32_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_int64_as_array.msg b/tests/opt_as_array/msg/test_opt_int64_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_int64_as_array.msg rename to tests/opt_as_array/msg/test_opt_int64_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_int8_as_array.msg b/tests/opt_as_array/msg/test_opt_int8_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_int8_as_array.msg rename to tests/opt_as_array/msg/test_opt_int8_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_std_empty_as_array.msg b/tests/opt_as_array/msg/test_opt_std_empty_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_std_empty_as_array.msg rename to tests/opt_as_array/msg/test_opt_std_empty_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_string_as_array.msg b/tests/opt_as_array/msg/test_opt_string_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_string_as_array.msg rename to tests/opt_as_array/msg/test_opt_string_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_time_as_array.msg b/tests/opt_as_array/msg/test_opt_time_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_time_as_array.msg rename to tests/opt_as_array/msg/test_opt_time_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_uint16_as_array.msg b/tests/opt_as_array/msg/test_opt_uint16_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_uint16_as_array.msg rename to tests/opt_as_array/msg/test_opt_uint16_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_uint32_as_array.msg b/tests/opt_as_array/msg/test_opt_uint32_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_uint32_as_array.msg rename to tests/opt_as_array/msg/test_opt_uint32_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_uint64_as_array.msg b/tests/opt_as_array/msg/test_opt_uint64_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_uint64_as_array.msg rename to tests/opt_as_array/msg/test_opt_uint64_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/msg/test_opt_uint8_as_array.msg b/tests/opt_as_array/msg/test_opt_uint8_as_array.msg similarity index 100% rename from pyros_msgs/opt_as_array/tests/msg/test_opt_uint8_as_array.msg rename to tests/opt_as_array/msg/test_opt_uint8_as_array.msg diff --git a/pyros_msgs/opt_as_array/tests/test_opt_bool.py b/tests/opt_as_array/test_opt_bool.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_bool.py rename to tests/opt_as_array/test_opt_bool.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_duration.py b/tests/opt_as_array/test_opt_duration.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_duration.py rename to tests/opt_as_array/test_opt_duration.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int16.py b/tests/opt_as_array/test_opt_int16.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_int16.py rename to tests/opt_as_array/test_opt_int16.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int32.py b/tests/opt_as_array/test_opt_int32.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_int32.py rename to tests/opt_as_array/test_opt_int32.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int64.py b/tests/opt_as_array/test_opt_int64.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_int64.py rename to tests/opt_as_array/test_opt_int64.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_int8.py b/tests/opt_as_array/test_opt_int8.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_int8.py rename to tests/opt_as_array/test_opt_int8.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_std_empty.py b/tests/opt_as_array/test_opt_std_empty.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_std_empty.py rename to tests/opt_as_array/test_opt_std_empty.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_string.py b/tests/opt_as_array/test_opt_string.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_string.py rename to tests/opt_as_array/test_opt_string.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_time.py b/tests/opt_as_array/test_opt_time.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_time.py rename to tests/opt_as_array/test_opt_time.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint16.py b/tests/opt_as_array/test_opt_uint16.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_uint16.py rename to tests/opt_as_array/test_opt_uint16.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint32.py b/tests/opt_as_array/test_opt_uint32.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_uint32.py rename to tests/opt_as_array/test_opt_uint32.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint64.py b/tests/opt_as_array/test_opt_uint64.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_uint64.py rename to tests/opt_as_array/test_opt_uint64.py diff --git a/pyros_msgs/opt_as_array/tests/test_opt_uint8.py b/tests/opt_as_array/test_opt_uint8.py similarity index 100% rename from pyros_msgs/opt_as_array/tests/test_opt_uint8.py rename to tests/opt_as_array/test_opt_uint8.py diff --git a/tests/opt_as_nested/__init__.py b/tests/opt_as_nested/__init__.py new file mode 100644 index 0000000..059dc31 --- /dev/null +++ b/tests/opt_as_nested/__init__.py @@ -0,0 +1,21 @@ +from __future__ import absolute_import, division, print_function + +import os +import sys +import site + +# This is used for message definitions, not for python code +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps')) + +# TMP : waiting for proper pip installable version of genpy and genmsd +try: + # Using genpy directly if ROS has been setup (while using from ROS pkg) + import genpy, genmsg +except ImportError: + # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) + import site + site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'ros-site')) + import genpy, genmsg + +import rosimport +rosimport.activate() diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_bool_as_nested.msg b/tests/opt_as_nested/msg/test_opt_bool_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_bool_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_bool_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_duration_as_nested.msg b/tests/opt_as_nested/msg/test_opt_duration_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_duration_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_duration_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_int16_as_nested.msg b/tests/opt_as_nested/msg/test_opt_int16_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_int16_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_int16_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_int32_as_nested.msg b/tests/opt_as_nested/msg/test_opt_int32_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_int32_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_int32_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_int64_as_nested.msg b/tests/opt_as_nested/msg/test_opt_int64_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_int64_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_int64_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_int8_as_nested.msg b/tests/opt_as_nested/msg/test_opt_int8_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_int8_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_int8_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_std_empty_as_nested.msg b/tests/opt_as_nested/msg/test_opt_std_empty_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_std_empty_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_std_empty_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_string_as_nested.msg b/tests/opt_as_nested/msg/test_opt_string_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_string_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_string_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_time_as_nested.msg b/tests/opt_as_nested/msg/test_opt_time_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_time_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_time_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_uint16_as_nested.msg b/tests/opt_as_nested/msg/test_opt_uint16_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_uint16_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_uint16_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_uint32_as_nested.msg b/tests/opt_as_nested/msg/test_opt_uint32_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_uint32_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_uint32_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_uint64_as_nested.msg b/tests/opt_as_nested/msg/test_opt_uint64_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_uint64_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_uint64_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/msg/test_opt_uint8_as_nested.msg b/tests/opt_as_nested/msg/test_opt_uint8_as_nested.msg similarity index 100% rename from pyros_msgs/opt_as_nested/tests/msg/test_opt_uint8_as_nested.msg rename to tests/opt_as_nested/msg/test_opt_uint8_as_nested.msg diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_bool.py b/tests/opt_as_nested/test_opt_bool.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_bool.py rename to tests/opt_as_nested/test_opt_bool.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_duration.py b/tests/opt_as_nested/test_opt_duration.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_duration.py rename to tests/opt_as_nested/test_opt_duration.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int16.py b/tests/opt_as_nested/test_opt_int16.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_int16.py rename to tests/opt_as_nested/test_opt_int16.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int32.py b/tests/opt_as_nested/test_opt_int32.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_int32.py rename to tests/opt_as_nested/test_opt_int32.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int64.py b/tests/opt_as_nested/test_opt_int64.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_int64.py rename to tests/opt_as_nested/test_opt_int64.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_int8.py b/tests/opt_as_nested/test_opt_int8.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_int8.py rename to tests/opt_as_nested/test_opt_int8.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py b/tests/opt_as_nested/test_opt_std_empty.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_std_empty.py rename to tests/opt_as_nested/test_opt_std_empty.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_string.py b/tests/opt_as_nested/test_opt_string.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_string.py rename to tests/opt_as_nested/test_opt_string.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_time.py b/tests/opt_as_nested/test_opt_time.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_time.py rename to tests/opt_as_nested/test_opt_time.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint16.py b/tests/opt_as_nested/test_opt_uint16.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_uint16.py rename to tests/opt_as_nested/test_opt_uint16.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint32.py b/tests/opt_as_nested/test_opt_uint32.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_uint32.py rename to tests/opt_as_nested/test_opt_uint32.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint64.py b/tests/opt_as_nested/test_opt_uint64.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_uint64.py rename to tests/opt_as_nested/test_opt_uint64.py diff --git a/pyros_msgs/opt_as_nested/tests/test_opt_uint8.py b/tests/opt_as_nested/test_opt_uint8.py similarity index 100% rename from pyros_msgs/opt_as_nested/tests/test_opt_uint8.py rename to tests/opt_as_nested/test_opt_uint8.py diff --git a/pyros_msgs/typecheck/tests/__init__.py b/tests/typecheck/__init__.py similarity index 100% rename from pyros_msgs/typecheck/tests/__init__.py rename to tests/typecheck/__init__.py diff --git a/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py b/tests/typecheck/test_basic_ros_serialization.py similarity index 96% rename from pyros_msgs/typecheck/tests/test_basic_ros_serialization.py rename to tests/typecheck/test_basic_ros_serialization.py index 08a5b90..4372c19 100644 --- a/pyros_msgs/typecheck/tests/test_basic_ros_serialization.py +++ b/tests/typecheck/test_basic_ros_serialization.py @@ -8,7 +8,7 @@ import site # This is used for message definitions, not for python code -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps')) import rosimport rosimport.activate() diff --git a/pyros_msgs/typecheck/tests/test_ros_mappings.py b/tests/typecheck/test_ros_mappings.py similarity index 99% rename from pyros_msgs/typecheck/tests/test_ros_mappings.py rename to tests/typecheck/test_ros_mappings.py index 784cce7..ad08da9 100644 --- a/pyros_msgs/typecheck/tests/test_ros_mappings.py +++ b/tests/typecheck/test_ros_mappings.py @@ -8,7 +8,7 @@ import site # This is used for message definitions, not for python code -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))), 'rosdeps')) +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps')) import rosimport rosimport.activate() diff --git a/pyros_msgs/typecheck/tests/test_typechecker_arrays.py b/tests/typecheck/test_typechecker_arrays.py similarity index 100% rename from pyros_msgs/typecheck/tests/test_typechecker_arrays.py rename to tests/typecheck/test_typechecker_arrays.py diff --git a/pyros_msgs/typecheck/tests/test_typechecker_basic.py b/tests/typecheck/test_typechecker_basic.py similarity index 100% rename from pyros_msgs/typecheck/tests/test_typechecker_basic.py rename to tests/typecheck/test_typechecker_basic.py diff --git a/pyros_msgs/typecheck/tests/test_typechecker_nested.py b/tests/typecheck/test_typechecker_nested.py similarity index 100% rename from pyros_msgs/typecheck/tests/test_typechecker_nested.py rename to tests/typecheck/test_typechecker_nested.py diff --git a/tox.ini b/tox.ini index 8767d59..77e9e18 100644 --- a/tox.ini +++ b/tox.ini @@ -62,7 +62,5 @@ setenv= commands= # we want to make sure python finds the installed package in tox env # and doesn't confuse with pyc generated during dev (which happens if we use self test feature here) - py.test --pyargs pyros_msgs/typecheck/tests {posargs} - py.test --pyargs pyros_msgs/opt_as_array/tests {posargs} - py.test --pyargs pyros_msgs/opt_as_nested/tests {posargs} + py.test --pyargs tests {posargs} # Note : -s here might break your terminal... From 3977350591ee1d72cc4e86f74466167938a431f1 Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 18 Jul 2017 16:06:01 +0900 Subject: [PATCH 21/28] fixing hypothesis deprecation warnings. --- tests/typecheck/test_ros_mappings.py | 55 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/tests/typecheck/test_ros_mappings.py b/tests/typecheck/test_ros_mappings.py index ad08da9..a2b0085 100644 --- a/tests/typecheck/test_ros_mappings.py +++ b/tests/typecheck/test_ros_mappings.py @@ -4,6 +4,7 @@ import sys import numpy import pytest +import collections from six import BytesIO import site @@ -54,42 +55,42 @@ # TODO : is there a way to generate messages on the fly to test all possible field combinations ? # For now We use a set of basic messages for testing -std_msgs_types_strat_ok = { - 'std_msgs/Bool': st.builds(std_msgs.Bool, data=st.booleans()), - 'std_msgs/Int8': st.builds(std_msgs.Int8, data=st.one_of(st.booleans(), st.integers(min_value=-128, max_value=127))), # in python booleans are integers - 'std_msgs/Int16': st.builds(std_msgs.Int16, data=st.one_of(st.booleans(), st.integers(min_value=-32768, max_value=32767))), - 'std_msgs/Int32': st.builds(std_msgs.Int32, data=st.one_of(st.booleans(), st.integers(min_value=-2147483648, max_value=2147483647))), - 'std_msgs/Int64': st.builds(std_msgs.Int64, data=st.one_of(st.booleans(), st.integers(min_value=-six_long(9223372036854775808), max_value=six_long(9223372036854775807)))), - 'std_msgs/UInt8': st.builds(std_msgs.UInt8, data=st.one_of(st.booleans(), st.integers(min_value=0, max_value=255))), - 'std_msgs/UInt16': st.builds(std_msgs.UInt16, data=st.one_of(st.booleans(), st.integers(min_value=0, max_value=65535))), - 'std_msgs/UInt32': st.builds(std_msgs.UInt32, data=st.one_of(st.booleans(), st.integers(min_value=0, max_value=4294967295))), - 'std_msgs/UInt64': st.builds(std_msgs.UInt64, data=st.one_of(st.booleans(), st.integers(min_value=0, max_value=six_long(18446744073709551615)))), +std_msgs_types_strat_ok = collections.OrderedDict([ + ('std_msgs/Bool', st.builds(std_msgs.Bool, data=st.booleans())), + ('std_msgs/Int8', st.builds(std_msgs.Int8, data=st.one_of(st.booleans(), st.integers(min_value=-128, max_value=127)))), # in python booleans are integers + ('std_msgs/Int16', st.builds(std_msgs.Int16, data=st.one_of(st.booleans(), st.integers(min_value=-32768, max_value=32767)))), + ('std_msgs/Int32', st.builds(std_msgs.Int32, data=st.one_of(st.booleans(), st.integers(min_value=-2147483648, max_value=2147483647)))), + ('std_msgs/Int64', st.builds(std_msgs.Int64, data=st.one_of(st.booleans(), st.integers(min_value=-six_long(9223372036854775808), max_value=six_long(9223372036854775807))))), + ('std_msgs/UInt8', st.builds(std_msgs.UInt8, data=st.one_of(st.booleans(), st.integers(min_value=0, max_value=255)))), + ('std_msgs/UInt16', st.builds(std_msgs.UInt16, data=st.one_of(st.booleans(), st.integers(min_value=0, max_value=65535)))), + ('std_msgs/UInt32', st.builds(std_msgs.UInt32, data=st.one_of(st.booleans(), st.integers(min_value=0, max_value=4294967295)))), + ('std_msgs/UInt64', st.builds(std_msgs.UInt64, data=st.one_of(st.booleans(), st.integers(min_value=0, max_value=six_long(18446744073709551615))))), # TMP : seems we have some problems with float arithmetic between numpy and hypothesis... #'std_msgs/Float32': st.builds(std_msgs.Float32, data=st.floats(min_value=-3.4028235e+38, max_value=3.4028235e+38)), #'std_msgs/Float64': st.builds(std_msgs.Float64, data=st.floats(min_value=-1.7976931348623157e+308, max_value=1.7976931348623157e+308, )), #'std_msgs/String': st.builds(std_msgs.String, data=st.one_of(st.binary(), st.text(alphabet=st.characters(max_codepoint=127)))), - 'std_msgs/String': st.builds(std_msgs.String, data=st.text(alphabet=st.characters(max_codepoint=127))), # binary can break hypothesis reporting + ('std_msgs/String', st.builds(std_msgs.String, data=st.text(alphabet=st.characters(max_codepoint=127)))), # binary can break hypothesis reporting # CAREFUL : we need to avoid having nsecs making our secs overflow after canonization from __init__ - 'std_msgs/Time': st.builds(std_msgs.Time, data=st.builds(genpy.Time, secs=st.integers(min_value=0, max_value=4294967295 -3), nsecs=st.integers(min_value=0, max_value=4294967295))), - 'std_msgs/Duration': st.builds(std_msgs.Duration, data=st.builds(genpy.Duration, secs=st.integers(min_value=-2147483648 +1, max_value=2147483647 -1), nsecs=st.integers(min_value=-2147483648, max_value=2147483647))), + ('std_msgs/Time', st.builds(std_msgs.Time, data=st.builds(genpy.Time, secs=st.integers(min_value=0, max_value=4294967295 -3), nsecs=st.integers(min_value=0, max_value=4294967295)))), + ('std_msgs/Duration', st.builds(std_msgs.Duration, data=st.builds(genpy.Duration, secs=st.integers(min_value=-2147483648 +1, max_value=2147483647 -1), nsecs=st.integers(min_value=-2147483648, max_value=2147483647)))), # TODO : add more. we should test all. -} +]) -std_msgs_types_strat_broken = { +std_msgs_types_strat_broken = collections.OrderedDict([ # everything else... - 'std_msgs/Bool': st.builds(std_msgs.Bool, data=st.one_of(st.integers(), st.floats())), - 'std_msgs/Int8': st.builds(std_msgs.Int8, data=st.one_of(st.floats(), st.integers(min_value=127+1), st.integers(max_value=-128-1))), - 'std_msgs/Int16': st.builds(std_msgs.Int16, data=st.one_of(st.floats(), st.integers(min_value=32767+1), st.integers(max_value=-32768-1))), - 'std_msgs/Int32': st.builds(std_msgs.Int32, data=st.one_of(st.floats(), st.integers(min_value=2147483647+1), st.integers(max_value=-2147483648-1))), - 'std_msgs/Int64': st.builds(std_msgs.Int64, data=st.one_of(st.floats(), st.integers(min_value=six_long(9223372036854775807+1)), st.integers(max_value=six_long(-9223372036854775808-1)))), - 'std_msgs/UInt8': st.builds(std_msgs.UInt8, data=st.one_of(st.floats(), st.integers(min_value=255+1), st.integers(max_value=-1))), - 'std_msgs/UInt16': st.builds(std_msgs.UInt16, data=st.one_of(st.floats(), st.integers(min_value=65535+1), st.integers(max_value=-1))), - 'std_msgs/UInt32': st.builds(std_msgs.UInt32, data=st.one_of(st.floats(), st.integers(min_value=4294967295+1), st.integers(max_value=-1))), - 'std_msgs/UInt64': st.builds(std_msgs.UInt64, data=st.one_of(st.floats(), st.integers(min_value=six_long(18446744073709551615+1)), st.integers(max_value=-1))), - 'std_msgs/Float32': st.builds(std_msgs.Float32, data=st.one_of(st.booleans(), st.integers())), # st.floats(max_value=-3.4028235e+38), st.floats(min_value=3.4028235e+38))), - 'std_msgs/Float64': st.builds(std_msgs.Float64, data=st.one_of(st.booleans(), st.integers())), # st.floats(max_value=-1.7976931348623157e+308), st.floats(min_value=1.7976931348623157e+308))), + ('std_msgs/Bool', st.builds(std_msgs.Bool, data=st.one_of(st.integers(), st.floats()))), + ('std_msgs/Int8', st.builds(std_msgs.Int8, data=st.one_of(st.floats(), st.integers(min_value=127+1), st.integers(max_value=-128-1)))), + ('std_msgs/Int16', st.builds(std_msgs.Int16, data=st.one_of(st.floats(), st.integers(min_value=32767+1), st.integers(max_value=-32768-1)))), + ('std_msgs/Int32', st.builds(std_msgs.Int32, data=st.one_of(st.floats(), st.integers(min_value=2147483647+1), st.integers(max_value=-2147483648-1)))), + ('std_msgs/Int64', st.builds(std_msgs.Int64, data=st.one_of(st.floats(), st.integers(min_value=six_long(9223372036854775807+1)), st.integers(max_value=six_long(-9223372036854775808-1))))), + ('std_msgs/UInt8', st.builds(std_msgs.UInt8, data=st.one_of(st.floats(), st.integers(min_value=255+1), st.integers(max_value=-1)))), + ('std_msgs/UInt16', st.builds(std_msgs.UInt16, data=st.one_of(st.floats(), st.integers(min_value=65535+1), st.integers(max_value=-1)))), + ('std_msgs/UInt32', st.builds(std_msgs.UInt32, data=st.one_of(st.floats(), st.integers(min_value=4294967295+1), st.integers(max_value=-1)))), + ('std_msgs/UInt64', st.builds(std_msgs.UInt64, data=st.one_of(st.floats(), st.integers(min_value=six_long(18446744073709551615+1)), st.integers(max_value=-1)))), + ('std_msgs/Float32', st.builds(std_msgs.Float32, data=st.one_of(st.booleans(), st.integers()))), # st.floats(max_value=-3.4028235e+38), st.floats(min_value=3.4028235e+38))), + ('std_msgs/Float64', st.builds(std_msgs.Float64, data=st.one_of(st.booleans(), st.integers()))), # st.floats(max_value=-1.7976931348623157e+308), st.floats(min_value=1.7976931348623157e+308))), # TODO : add more. we should test all -} +]) pyros_msgs_types_strat_ok = { # TODO From 1d8652aead6e41be8b37c1e326965216d28a54ae Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 18 Jul 2017 17:39:34 +0900 Subject: [PATCH 22/28] removing rosimport submodule since it has been released. --- .gitmodules | 3 --- pydeps/rosimport | 1 - ros-site/rosimport.pth | 1 - 3 files changed, 5 deletions(-) delete mode 160000 pydeps/rosimport delete mode 100644 ros-site/rosimport.pth diff --git a/.gitmodules b/.gitmodules index aaf315d..2351cfc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,6 +10,3 @@ [submodule "rosdeps/ros_comm_msgs"] path = rosdeps/ros_comm_msgs url = https://github.com/ros/ros_comm_msgs.git -[submodule "pydeps/rosimport"] - path = pydeps/rosimport - url = https://github.com/asmodehn/rosimport diff --git a/pydeps/rosimport b/pydeps/rosimport deleted file mode 160000 index e806ef2..0000000 --- a/pydeps/rosimport +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e806ef2a05279a6d0b86b0ffa577a92048272c5c diff --git a/ros-site/rosimport.pth b/ros-site/rosimport.pth deleted file mode 100644 index dfc2d75..0000000 --- a/ros-site/rosimport.pth +++ /dev/null @@ -1 +0,0 @@ -../pydeps/rosimport \ No newline at end of file From e4a8f3e16eb754fa971a9ea51b73d1e677baf5bc Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 18 Jul 2017 18:51:11 +0900 Subject: [PATCH 23/28] attempting to fix setup.py to include .msg --- README.md | 13 ------------- README.rst | 21 +++++++++++++++++++++ dev-requirements.txt | 4 ++++ setup.py | 20 +++++++++----------- 4 files changed, 34 insertions(+), 24 deletions(-) delete mode 100644 README.md create mode 100644 README.rst diff --git a/README.md b/README.md deleted file mode 100644 index bd2783c..0000000 --- a/README.md +++ /dev/null @@ -1,13 +0,0 @@ -[![Code Issues](https://www.quantifiedcode.com/api/v1/project/646d87c377144f1fa5c9a328a883c619/badge.svg)](https://www.quantifiedcode.com/app/project/646d87c377144f1fa5c9a328a883c619) -[![Build Status](https://travis-ci.org/asmodehn/pyros-msgs.svg?branch=master)](https://travis-ci.org/asmodehn/pyros-msgs) - -# Pyros-msgs - -ROS Package enabling ROS communication for other Pyros multiprocess systems. - -## Features - -### ROS -- optional field as a ROS array -- optional field as a specific message type (Work In Progress) - diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..f5e357a --- /dev/null +++ b/README.rst @@ -0,0 +1,21 @@ +|Code Issues| |Build Status| + +Pyros-msgs +========== + +ROS Package enabling ROS communication for other Pyros multiprocess +systems. + +Features +-------- + +ROS +~~~ + +- optional field as a ROS array +- optional field as a specific message type (Work In Progress) + +.. |Code Issues| image:: https://www.quantifiedcode.com/api/v1/project/646d87c377144f1fa5c9a328a883c619/badge.svg + :target: https://www.quantifiedcode.com/app/project/646d87c377144f1fa5c9a328a883c619 +.. |Build Status| image:: https://travis-ci.org/asmodehn/pyros-msgs.svg?branch=master + :target: https://travis-ci.org/asmodehn/pyros-msgs \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index 093784a..31998c0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,9 @@ gitchangelog twine +pytest>=2.8.0, # as per hypothesis requirement (careful with 2.5.1 on trusty) +pytest-xdist, # for --boxed (careful with the version it will be moved out of xdist) +hypothesis>=3.0.1, # to target xenial LTS version +numpy>=1.8.2, # from trusty version rosimport # source access to latest filefinder from git ... diff --git a/setup.py b/setup.py index 8120564..8228392 100644 --- a/setup.py +++ b/setup.py @@ -196,35 +196,33 @@ def run(self): license='MIT', packages=[ 'pyros_msgs', - # 'pyros_msgs.msg', #catkin build should generate it anyway. python build can assume client uses rosimport. + 'pyros_msgs.msg', # catkin build should generate it. python build can assume client uses rosimport. 'pyros_msgs.typecheck', 'pyros_msgs.opt_as_array', 'pyros_msgs.opt_as_nested', ], + package_data={ + '': ['*.msg', '*.srv'] + }, namespace_packages=['pyros_msgs'], # this is better than using package data ( since behavior is a bit different from distutils... ) - include_package_data=True, # use MANIFEST.in during install. + #include_package_data=True, # use MANIFEST.in during install. # Reference for optional dependencies : http://stackoverflow.com/questions/4796936/does-pip-handle-extras-requires-from-setuptools-distribute-based-sources install_requires=[ 'six', - # this is needed as install dependency since we embed tests in the package. - # 'pyros_setup>=0.2.1', # needed to grab ros environment even if distro setup.sh not sourced - # 'pyros_utils', # this must be satisfied by the ROS package system... - # 'importlib2>=3.4;python_version<"3.4"', # NOT working we use a patched version of it, through a symlink (to make linux deb release possible) - 'filefinder2; python_version<"3.4"', # we rely on this for PEP420 on python 2.7 'pyyaml>=3.10', # genpy relies on this... + ], + tests_require=[ + 'rosimport', # we rely on this for generating ROS message if necessary before importing 'pytest>=2.8.0', # as per hypothesis requirement (careful with 2.5.1 on trusty) 'pytest-xdist', # for --boxed (careful with the version it will be moved out of xdist) 'hypothesis>=3.0.1', # to target xenial LTS version 'numpy>=1.8.2', # from trusty version ], - test_requires=[ - 'rosimport', # we rely on this for generating ROS message if necessary before importing - ], cmdclass={ 'rosdevelop': RosDevelopCommand, 'rospublish': ROSPublishCommand, }, - zip_safe=False, # TODO testing... + zip_safe=False, # TODO testing... including using rosimport to generate code from message definition... ) From a44105290c46f26cad2790437319fbf443df761a Mon Sep 17 00:00:00 2001 From: alexv Date: Thu, 27 Jul 2017 18:34:29 +0900 Subject: [PATCH 24/28] restructuring to extract tests, fix package dist, replace ros submodules with pips --- .gitignore | 5 +++++ .gitmodules | 14 ++++--------- MANIFEST.in | 3 +++ dev-requirements.txt | 19 ++++++++++------- msg | 1 + pyros_msgs/__init__.py | 15 ------------- {msg => pyros_msgs/msg}/OptionalFields.msg | 0 pyros_msgs/opt_as_nested/opt_as_nested.py | 4 ---- pyros_msgs/typecheck/ros_mappings.py | 2 +- ros-site/README.md | 6 ------ ros-site/genmsg.pth | 1 - ros-site/genpy.pth | 1 - rosdeps/genmsg | 1 - rosdeps/genpy | 1 - setup.py | 2 +- tests/opt_as_array/__init__.py | 21 ------------------- tests/opt_as_nested/__init__.py | 21 ------------------- {rosdeps => tests/rosdeps}/ros_comm_msgs | 0 {rosdeps => tests/rosdeps}/std_msgs | 0 tests/{ => test_pyros_msgs}/__init__.py | 0 .../test_pyros_msgs/opt_as_array/__init__.py | 17 +++++++++++++++ .../msg/test_opt_bool_as_array.msg | 0 .../msg/test_opt_duration_as_array.msg | 0 .../msg/test_opt_float32_as_array.msg | 0 .../msg/test_opt_float64_as_array.msg | 0 .../msg/test_opt_header_as_array.msg | 0 .../msg/test_opt_int16_as_array.msg | 0 .../msg/test_opt_int32_as_array.msg | 0 .../msg/test_opt_int64_as_array.msg | 0 .../msg/test_opt_int8_as_array.msg | 0 .../msg/test_opt_std_empty_as_array.msg | 0 .../msg/test_opt_string_as_array.msg | 0 .../msg/test_opt_time_as_array.msg | 0 .../msg/test_opt_uint16_as_array.msg | 0 .../msg/test_opt_uint32_as_array.msg | 0 .../msg/test_opt_uint64_as_array.msg | 0 .../msg/test_opt_uint8_as_array.msg | 0 .../opt_as_array/test_opt_bool.py | 0 .../opt_as_array/test_opt_duration.py | 0 .../opt_as_array/test_opt_int16.py | 0 .../opt_as_array/test_opt_int32.py | 0 .../opt_as_array/test_opt_int64.py | 0 .../opt_as_array/test_opt_int8.py | 0 .../opt_as_array/test_opt_std_empty.py | 0 .../opt_as_array/test_opt_string.py | 0 .../opt_as_array/test_opt_time.py | 0 .../opt_as_array/test_opt_uint16.py | 0 .../opt_as_array/test_opt_uint32.py | 0 .../opt_as_array/test_opt_uint64.py | 0 .../opt_as_array/test_opt_uint8.py | 0 .../test_pyros_msgs/opt_as_nested/__init__.py | 14 +++++++++++++ .../msg/test_opt_bool_as_nested.msg | 0 .../msg/test_opt_duration_as_nested.msg | 0 .../msg/test_opt_int16_as_nested.msg | 0 .../msg/test_opt_int32_as_nested.msg | 0 .../msg/test_opt_int64_as_nested.msg | 0 .../msg/test_opt_int8_as_nested.msg | 0 .../msg/test_opt_std_empty_as_nested.msg | 0 .../msg/test_opt_string_as_nested.msg | 0 .../msg/test_opt_time_as_nested.msg | 0 .../msg/test_opt_uint16_as_nested.msg | 0 .../msg/test_opt_uint32_as_nested.msg | 0 .../msg/test_opt_uint64_as_nested.msg | 0 .../msg/test_opt_uint8_as_nested.msg | 0 .../opt_as_nested/test_opt_bool.py | 0 .../opt_as_nested/test_opt_duration.py | 0 .../opt_as_nested/test_opt_int16.py | 0 .../opt_as_nested/test_opt_int32.py | 0 .../opt_as_nested/test_opt_int64.py | 0 .../opt_as_nested/test_opt_int8.py | 0 .../opt_as_nested/test_opt_std_empty.py | 0 .../opt_as_nested/test_opt_string.py | 0 .../opt_as_nested/test_opt_time.py | 0 .../opt_as_nested/test_opt_uint16.py | 0 .../opt_as_nested/test_opt_uint32.py | 0 .../opt_as_nested/test_opt_uint64.py | 0 .../opt_as_nested/test_opt_uint8.py | 0 .../typecheck/__init__.py | 10 --------- .../typecheck/test_basic_ros_serialization.py | 0 .../typecheck/test_ros_mappings.py | 0 .../typecheck/test_typechecker_arrays.py | 0 .../typecheck/test_typechecker_basic.py | 0 .../typecheck/test_typechecker_nested.py | 0 tox.ini | 12 +++++------ 84 files changed, 63 insertions(+), 107 deletions(-) create mode 100644 MANIFEST.in create mode 120000 msg rename {msg => pyros_msgs/msg}/OptionalFields.msg (100%) delete mode 100644 ros-site/README.md delete mode 100644 ros-site/genmsg.pth delete mode 100644 ros-site/genpy.pth delete mode 160000 rosdeps/genmsg delete mode 160000 rosdeps/genpy delete mode 100644 tests/opt_as_array/__init__.py delete mode 100644 tests/opt_as_nested/__init__.py rename {rosdeps => tests/rosdeps}/ros_comm_msgs (100%) rename {rosdeps => tests/rosdeps}/std_msgs (100%) rename tests/{ => test_pyros_msgs}/__init__.py (100%) create mode 100644 tests/test_pyros_msgs/opt_as_array/__init__.py rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_bool_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_duration_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_float32_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_float64_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_header_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_int16_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_int32_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_int64_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_int8_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_std_empty_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_string_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_time_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_uint16_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_uint32_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_uint64_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/msg/test_opt_uint8_as_array.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_bool.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_duration.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_int16.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_int32.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_int64.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_int8.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_std_empty.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_string.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_time.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_uint16.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_uint32.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_uint64.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_array/test_opt_uint8.py (100%) create mode 100644 tests/test_pyros_msgs/opt_as_nested/__init__.py rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_bool_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_duration_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_int16_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_int32_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_int64_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_int8_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_std_empty_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_string_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_time_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_uint16_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_uint32_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_uint64_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/msg/test_opt_uint8_as_nested.msg (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_bool.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_duration.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_int16.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_int32.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_int64.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_int8.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_std_empty.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_string.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_time.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_uint16.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_uint32.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_uint64.py (100%) rename tests/{ => test_pyros_msgs}/opt_as_nested/test_opt_uint8.py (100%) rename tests/{ => test_pyros_msgs}/typecheck/__init__.py (97%) rename tests/{ => test_pyros_msgs}/typecheck/test_basic_ros_serialization.py (100%) rename tests/{ => test_pyros_msgs}/typecheck/test_ros_mappings.py (100%) rename tests/{ => test_pyros_msgs}/typecheck/test_typechecker_arrays.py (100%) rename tests/{ => test_pyros_msgs}/typecheck/test_typechecker_basic.py (100%) rename tests/{ => test_pyros_msgs}/typecheck/test_typechecker_nested.py (100%) diff --git a/.gitignore b/.gitignore index dc106bf..c3cebe4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,10 @@ *.pyc build +dist .tox .hypothesis *.egg-info +.cache +.idea +__pycache__ + diff --git a/.gitmodules b/.gitmodules index 2351cfc..0922419 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,6 @@ -[submodule "rosdeps/genmsg"] - path = rosdeps/genmsg - url = https://github.com/ros/genmsg.git -[submodule "rosdeps/genpy"] - path = rosdeps/genpy - url = https://github.com/ros/genpy.git -[submodule "rosdeps/std_msgs"] - path = rosdeps/std_msgs +[submodule "tests/rosdeps/std_msgs"] + path = tests/rosdeps/std_msgs url = https://github.com/ros/std_msgs.git -[submodule "rosdeps/ros_comm_msgs"] - path = rosdeps/ros_comm_msgs +[submodule "tests/rosdeps/ros_comm_msgs"] + path = tests/rosdeps/ros_comm_msgs url = https://github.com/ros/ros_comm_msgs.git diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..b570fbf --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +# default +include README.rst +include setup.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 31998c0..80fb301 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,11 +1,14 @@ gitchangelog twine -pytest>=2.8.0, # as per hypothesis requirement (careful with 2.5.1 on trusty) -pytest-xdist, # for --boxed (careful with the version it will be moved out of xdist) -hypothesis>=3.0.1, # to target xenial LTS version -numpy>=1.8.2, # from trusty version -rosimport +pytest>=2.8.0 # as per hypothesis requirement (careful with 2.5.1 on trusty) +pytest-xdist # for --boxed (careful with the version it will be moved out of xdist) +hypothesis>=3.0.1 # to target xenial LTS version +numpy>=1.8.2 # from trusty version -# source access to latest filefinder from git ... -#-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 -#-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport +# source access to latest filefinder and rosimport from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport + +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools#egg=ros_genpy diff --git a/msg b/msg new file mode 120000 index 0000000..7ad3823 --- /dev/null +++ b/msg @@ -0,0 +1 @@ +pyros_msgs/msg \ No newline at end of file diff --git a/pyros_msgs/__init__.py b/pyros_msgs/__init__.py index 62b0fcc..e69de29 100644 --- a/pyros_msgs/__init__.py +++ b/pyros_msgs/__init__.py @@ -1,15 +0,0 @@ -from __future__ import absolute_import, division, print_function -# This is a namespace package to merge the ROS generated messages and the python subpackage sources -# Note : this file must be loaded first in order for other subpackages to be found -# Note : this works for development packages only if pyros_setup has been activated -# because it will put egg directories on pythonpath FIRST. - -# Ref : https://packaging.python.org/namespace_packages/ - -# pkgutil does not seem, somehow, to be compatible with ROS devel space... -# import pkgutil -# __path__ = pkgutil.extend_path(__path__, __name__) - -# Be aware : https://github.com/jonparrott/namespace-pkg-tests/blob/master/table.md -import pkg_resources -pkg_resources.declare_namespace(__name__) diff --git a/msg/OptionalFields.msg b/pyros_msgs/msg/OptionalFields.msg similarity index 100% rename from msg/OptionalFields.msg rename to pyros_msgs/msg/OptionalFields.msg diff --git a/pyros_msgs/opt_as_nested/opt_as_nested.py b/pyros_msgs/opt_as_nested/opt_as_nested.py index 0937155..8aafe28 100644 --- a/pyros_msgs/opt_as_nested/opt_as_nested.py +++ b/pyros_msgs/opt_as_nested/opt_as_nested.py @@ -11,10 +11,6 @@ make_typechecker_field_hidden, ) -import os -import sys - - from .ros_mappings import typechecker_from_rosfield_opttype diff --git a/pyros_msgs/typecheck/ros_mappings.py b/pyros_msgs/typecheck/ros_mappings.py index 0492693..e1e397a 100644 --- a/pyros_msgs/typecheck/ros_mappings.py +++ b/pyros_msgs/typecheck/ros_mappings.py @@ -27,7 +27,7 @@ import six -from pyros_msgs.typecheck import ( +from .typechecker import ( six_long, Accepter, Sanitizer, Array, Any, MinMax, CodePoint, TypeChecker, diff --git a/ros-site/README.md b/ros-site/README.md deleted file mode 100644 index 24202b8..0000000 --- a/ros-site/README.md +++ /dev/null @@ -1,6 +0,0 @@ -This folder is added with site.addsitedir() to provide a set of ROS python packages from source. -It happens when the import of one of these packages(usually found on ROS systems) fails. -This is the case from pure python, for our automated testing tox suite. - -If ROS is setup before executing pyros_msgs/importer/rosmsg_generator.py, -then the ROS versions of these packages will be used instead \ No newline at end of file diff --git a/ros-site/genmsg.pth b/ros-site/genmsg.pth deleted file mode 100644 index 297013a..0000000 --- a/ros-site/genmsg.pth +++ /dev/null @@ -1 +0,0 @@ -../rosdeps/genmsg/src \ No newline at end of file diff --git a/ros-site/genpy.pth b/ros-site/genpy.pth deleted file mode 100644 index 79a5b28..0000000 --- a/ros-site/genpy.pth +++ /dev/null @@ -1 +0,0 @@ -../rosdeps/genpy/src \ No newline at end of file diff --git a/rosdeps/genmsg b/rosdeps/genmsg deleted file mode 160000 index 490b375..0000000 --- a/rosdeps/genmsg +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 490b3753213ed3e2251acd218c62224b2787ee0e diff --git a/rosdeps/genpy b/rosdeps/genpy deleted file mode 160000 index cb07134..0000000 --- a/rosdeps/genpy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cb07134e084a4fb6099de5cefcbb6b9a7f666cf5 diff --git a/setup.py b/setup.py index 8228392..3f49ec1 100644 --- a/setup.py +++ b/setup.py @@ -204,7 +204,6 @@ def run(self): package_data={ '': ['*.msg', '*.srv'] }, - namespace_packages=['pyros_msgs'], # this is better than using package data ( since behavior is a bit different from distutils... ) #include_package_data=True, # use MANIFEST.in during install. # Reference for optional dependencies : http://stackoverflow.com/questions/4796936/does-pip-handle-extras-requires-from-setuptools-distribute-based-sources @@ -213,6 +212,7 @@ def run(self): 'pyyaml>=3.10', # genpy relies on this... ], tests_require=[ + 'filefinder2', 'rosimport', # we rely on this for generating ROS message if necessary before importing 'pytest>=2.8.0', # as per hypothesis requirement (careful with 2.5.1 on trusty) 'pytest-xdist', # for --boxed (careful with the version it will be moved out of xdist) diff --git a/tests/opt_as_array/__init__.py b/tests/opt_as_array/__init__.py deleted file mode 100644 index d544393..0000000 --- a/tests/opt_as_array/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import os -import sys -import site - -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps')) -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'pydeps')) - -# TMP : waiting for proper pip installable version of genpy and genmsd -try: - # Using genpy directly if ROS has been setup (while using from ROS pkg) - import genpy, genmsg -except ImportError: - # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) - import site - site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'ros-site')) - import genpy, genmsg - -import rosimport -rosimport.activate() diff --git a/tests/opt_as_nested/__init__.py b/tests/opt_as_nested/__init__.py deleted file mode 100644 index 059dc31..0000000 --- a/tests/opt_as_nested/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import absolute_import, division, print_function - -import os -import sys -import site - -# This is used for message definitions, not for python code -site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps')) - -# TMP : waiting for proper pip installable version of genpy and genmsd -try: - # Using genpy directly if ROS has been setup (while using from ROS pkg) - import genpy, genmsg -except ImportError: - # Otherwise we refer to our submodules here (setup.py usecase, or running from tox without site-packages) - import site - site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'ros-site')) - import genpy, genmsg - -import rosimport -rosimport.activate() diff --git a/rosdeps/ros_comm_msgs b/tests/rosdeps/ros_comm_msgs similarity index 100% rename from rosdeps/ros_comm_msgs rename to tests/rosdeps/ros_comm_msgs diff --git a/rosdeps/std_msgs b/tests/rosdeps/std_msgs similarity index 100% rename from rosdeps/std_msgs rename to tests/rosdeps/std_msgs diff --git a/tests/__init__.py b/tests/test_pyros_msgs/__init__.py similarity index 100% rename from tests/__init__.py rename to tests/test_pyros_msgs/__init__.py diff --git a/tests/test_pyros_msgs/opt_as_array/__init__.py b/tests/test_pyros_msgs/opt_as_array/__init__.py new file mode 100644 index 0000000..1ecdaa8 --- /dev/null +++ b/tests/test_pyros_msgs/opt_as_array/__init__.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import, division, print_function + +import os +import sys +import site + +added_site_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps') +print("Adding site directory {0} to access genmsg, genpy and, std_msgs".format(added_site_dir)) +site.addsitedir(added_site_dir) + +# Using genpy directly if ROS has been setup (while using from ROS pkg) +import genpy, genmsg + +print(sys.path) + +import rosimport +rosimport.activate() diff --git a/tests/opt_as_array/msg/test_opt_bool_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_bool_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_bool_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_bool_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_duration_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_duration_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_duration_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_duration_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_float32_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_float32_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_float32_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_float32_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_float64_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_float64_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_float64_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_float64_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_header_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_header_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_header_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_header_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_int16_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_int16_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_int16_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_int16_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_int32_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_int32_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_int32_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_int32_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_int64_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_int64_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_int64_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_int64_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_int8_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_int8_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_int8_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_int8_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_std_empty_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_std_empty_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_std_empty_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_std_empty_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_string_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_string_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_string_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_string_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_time_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_time_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_time_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_time_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_uint16_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_uint16_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_uint16_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_uint16_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_uint32_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_uint32_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_uint32_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_uint32_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_uint64_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_uint64_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_uint64_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_uint64_as_array.msg diff --git a/tests/opt_as_array/msg/test_opt_uint8_as_array.msg b/tests/test_pyros_msgs/opt_as_array/msg/test_opt_uint8_as_array.msg similarity index 100% rename from tests/opt_as_array/msg/test_opt_uint8_as_array.msg rename to tests/test_pyros_msgs/opt_as_array/msg/test_opt_uint8_as_array.msg diff --git a/tests/opt_as_array/test_opt_bool.py b/tests/test_pyros_msgs/opt_as_array/test_opt_bool.py similarity index 100% rename from tests/opt_as_array/test_opt_bool.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_bool.py diff --git a/tests/opt_as_array/test_opt_duration.py b/tests/test_pyros_msgs/opt_as_array/test_opt_duration.py similarity index 100% rename from tests/opt_as_array/test_opt_duration.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_duration.py diff --git a/tests/opt_as_array/test_opt_int16.py b/tests/test_pyros_msgs/opt_as_array/test_opt_int16.py similarity index 100% rename from tests/opt_as_array/test_opt_int16.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_int16.py diff --git a/tests/opt_as_array/test_opt_int32.py b/tests/test_pyros_msgs/opt_as_array/test_opt_int32.py similarity index 100% rename from tests/opt_as_array/test_opt_int32.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_int32.py diff --git a/tests/opt_as_array/test_opt_int64.py b/tests/test_pyros_msgs/opt_as_array/test_opt_int64.py similarity index 100% rename from tests/opt_as_array/test_opt_int64.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_int64.py diff --git a/tests/opt_as_array/test_opt_int8.py b/tests/test_pyros_msgs/opt_as_array/test_opt_int8.py similarity index 100% rename from tests/opt_as_array/test_opt_int8.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_int8.py diff --git a/tests/opt_as_array/test_opt_std_empty.py b/tests/test_pyros_msgs/opt_as_array/test_opt_std_empty.py similarity index 100% rename from tests/opt_as_array/test_opt_std_empty.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_std_empty.py diff --git a/tests/opt_as_array/test_opt_string.py b/tests/test_pyros_msgs/opt_as_array/test_opt_string.py similarity index 100% rename from tests/opt_as_array/test_opt_string.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_string.py diff --git a/tests/opt_as_array/test_opt_time.py b/tests/test_pyros_msgs/opt_as_array/test_opt_time.py similarity index 100% rename from tests/opt_as_array/test_opt_time.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_time.py diff --git a/tests/opt_as_array/test_opt_uint16.py b/tests/test_pyros_msgs/opt_as_array/test_opt_uint16.py similarity index 100% rename from tests/opt_as_array/test_opt_uint16.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_uint16.py diff --git a/tests/opt_as_array/test_opt_uint32.py b/tests/test_pyros_msgs/opt_as_array/test_opt_uint32.py similarity index 100% rename from tests/opt_as_array/test_opt_uint32.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_uint32.py diff --git a/tests/opt_as_array/test_opt_uint64.py b/tests/test_pyros_msgs/opt_as_array/test_opt_uint64.py similarity index 100% rename from tests/opt_as_array/test_opt_uint64.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_uint64.py diff --git a/tests/opt_as_array/test_opt_uint8.py b/tests/test_pyros_msgs/opt_as_array/test_opt_uint8.py similarity index 100% rename from tests/opt_as_array/test_opt_uint8.py rename to tests/test_pyros_msgs/opt_as_array/test_opt_uint8.py diff --git a/tests/test_pyros_msgs/opt_as_nested/__init__.py b/tests/test_pyros_msgs/opt_as_nested/__init__.py new file mode 100644 index 0000000..dddba74 --- /dev/null +++ b/tests/test_pyros_msgs/opt_as_nested/__init__.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import, division, print_function + +import os +import sys +import site + +# This is used for message definitions, not for python code +site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps')) + +# Using genpy directly if ROS has been setup (while using from ROS pkg) +import genpy, genmsg + +import rosimport +rosimport.activate() diff --git a/tests/opt_as_nested/msg/test_opt_bool_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_bool_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_bool_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_bool_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_duration_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_duration_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_duration_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_duration_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_int16_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_int16_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_int16_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_int16_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_int32_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_int32_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_int32_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_int32_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_int64_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_int64_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_int64_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_int64_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_int8_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_int8_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_int8_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_int8_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_std_empty_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_std_empty_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_std_empty_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_std_empty_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_string_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_string_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_string_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_string_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_time_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_time_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_time_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_time_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_uint16_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_uint16_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_uint16_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_uint16_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_uint32_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_uint32_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_uint32_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_uint32_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_uint64_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_uint64_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_uint64_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_uint64_as_nested.msg diff --git a/tests/opt_as_nested/msg/test_opt_uint8_as_nested.msg b/tests/test_pyros_msgs/opt_as_nested/msg/test_opt_uint8_as_nested.msg similarity index 100% rename from tests/opt_as_nested/msg/test_opt_uint8_as_nested.msg rename to tests/test_pyros_msgs/opt_as_nested/msg/test_opt_uint8_as_nested.msg diff --git a/tests/opt_as_nested/test_opt_bool.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_bool.py similarity index 100% rename from tests/opt_as_nested/test_opt_bool.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_bool.py diff --git a/tests/opt_as_nested/test_opt_duration.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_duration.py similarity index 100% rename from tests/opt_as_nested/test_opt_duration.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_duration.py diff --git a/tests/opt_as_nested/test_opt_int16.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_int16.py similarity index 100% rename from tests/opt_as_nested/test_opt_int16.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_int16.py diff --git a/tests/opt_as_nested/test_opt_int32.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_int32.py similarity index 100% rename from tests/opt_as_nested/test_opt_int32.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_int32.py diff --git a/tests/opt_as_nested/test_opt_int64.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_int64.py similarity index 100% rename from tests/opt_as_nested/test_opt_int64.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_int64.py diff --git a/tests/opt_as_nested/test_opt_int8.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_int8.py similarity index 100% rename from tests/opt_as_nested/test_opt_int8.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_int8.py diff --git a/tests/opt_as_nested/test_opt_std_empty.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_std_empty.py similarity index 100% rename from tests/opt_as_nested/test_opt_std_empty.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_std_empty.py diff --git a/tests/opt_as_nested/test_opt_string.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_string.py similarity index 100% rename from tests/opt_as_nested/test_opt_string.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_string.py diff --git a/tests/opt_as_nested/test_opt_time.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_time.py similarity index 100% rename from tests/opt_as_nested/test_opt_time.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_time.py diff --git a/tests/opt_as_nested/test_opt_uint16.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint16.py similarity index 100% rename from tests/opt_as_nested/test_opt_uint16.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_uint16.py diff --git a/tests/opt_as_nested/test_opt_uint32.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint32.py similarity index 100% rename from tests/opt_as_nested/test_opt_uint32.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_uint32.py diff --git a/tests/opt_as_nested/test_opt_uint64.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint64.py similarity index 100% rename from tests/opt_as_nested/test_opt_uint64.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_uint64.py diff --git a/tests/opt_as_nested/test_opt_uint8.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint8.py similarity index 100% rename from tests/opt_as_nested/test_opt_uint8.py rename to tests/test_pyros_msgs/opt_as_nested/test_opt_uint8.py diff --git a/tests/typecheck/__init__.py b/tests/test_pyros_msgs/typecheck/__init__.py similarity index 97% rename from tests/typecheck/__init__.py rename to tests/test_pyros_msgs/typecheck/__init__.py index 2b7400e..c50a829 100644 --- a/tests/typecheck/__init__.py +++ b/tests/test_pyros_msgs/typecheck/__init__.py @@ -3,16 +3,6 @@ import pytest import sys -# try: -# import pyros_msgs -# import genpy -# except ImportError: -# import pyros_setup -# pyros_setup.configurable_import().configure().activate() -# import pyros_msgs -# import genpy - - import six from pyros_msgs.typecheck.typechecker import ( diff --git a/tests/typecheck/test_basic_ros_serialization.py b/tests/test_pyros_msgs/typecheck/test_basic_ros_serialization.py similarity index 100% rename from tests/typecheck/test_basic_ros_serialization.py rename to tests/test_pyros_msgs/typecheck/test_basic_ros_serialization.py diff --git a/tests/typecheck/test_ros_mappings.py b/tests/test_pyros_msgs/typecheck/test_ros_mappings.py similarity index 100% rename from tests/typecheck/test_ros_mappings.py rename to tests/test_pyros_msgs/typecheck/test_ros_mappings.py diff --git a/tests/typecheck/test_typechecker_arrays.py b/tests/test_pyros_msgs/typecheck/test_typechecker_arrays.py similarity index 100% rename from tests/typecheck/test_typechecker_arrays.py rename to tests/test_pyros_msgs/typecheck/test_typechecker_arrays.py diff --git a/tests/typecheck/test_typechecker_basic.py b/tests/test_pyros_msgs/typecheck/test_typechecker_basic.py similarity index 100% rename from tests/typecheck/test_typechecker_basic.py rename to tests/test_pyros_msgs/typecheck/test_typechecker_basic.py diff --git a/tests/typecheck/test_typechecker_nested.py b/tests/test_pyros_msgs/typecheck/test_typechecker_nested.py similarity index 100% rename from tests/typecheck/test_typechecker_nested.py rename to tests/test_pyros_msgs/typecheck/test_typechecker_nested.py diff --git a/tox.ini b/tox.ini index 77e9e18..9af67ee 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ envlist = py27 # we skip sdist since currently our sdist doesnt generate messages # we will manually setup the package via develop in [testenv] -skipsdist=True +#skipsdist=True [travis] 2.7 = py27 @@ -39,8 +39,8 @@ recreate=True # sitepackages=True # We do not want any access to system (same as basic travis python testing) -# TMP because our dependencies are not in pips -usedevelop = True +;# TMP because our dependencies are not in pips +;usedevelop = True # we need to set our ROS distro in the pythonpath for our code to rely on ROS #TODO : not havnig this breaks pyros-setup which tries to create its config file in @@ -55,9 +55,9 @@ usedevelop = True setenv= # TODO : get rid of this by removing all dependencies to ROS on system... - indigo: PYTHONPATH=/opt/ros/indigo/lib/python2.7/dist-packages - jade: PYTHONPATH=/opt/ros/jade/lib/python2.7/dist-packages - kinetic: PYTHONPATH=/opt/ros/kinetic/lib/python2.7/dist-packages +; indigo: PYTHONPATH=/opt/ros/indigo/lib/python2.7/dist-packages +; jade: PYTHONPATH=/opt/ros/jade/lib/python2.7/dist-packages +; kinetic: PYTHONPATH=/opt/ros/kinetic/lib/python2.7/dist-packages commands= # we want to make sure python finds the installed package in tox env From df3129c80a1e04e27f5d86758155e896d68e8e59 Mon Sep 17 00:00:00 2001 From: alexv Date: Mon, 7 Aug 2017 17:32:41 +0900 Subject: [PATCH 25/28] activating python3 support --- .travis.yml | 6 +-- dev-requirements.txt | 2 +- pyros_msgs/typecheck/typechecker.py | 32 +++++++++++++--- .../test_pyros_msgs/opt_as_array/__init__.py | 7 +--- .../opt_as_array/test_opt_string.py | 16 ++++++-- .../opt_as_nested/test_opt_bool.py | 2 +- .../opt_as_nested/test_opt_int16.py | 2 +- .../opt_as_nested/test_opt_int32.py | 2 +- .../opt_as_nested/test_opt_int64.py | 2 +- .../opt_as_nested/test_opt_int8.py | 2 +- .../opt_as_nested/test_opt_string.py | 16 ++++++-- .../opt_as_nested/test_opt_uint16.py | 2 +- .../opt_as_nested/test_opt_uint32.py | 2 +- .../opt_as_nested/test_opt_uint64.py | 2 +- .../opt_as_nested/test_opt_uint8.py | 2 +- .../typecheck/test_ros_mappings.py | 7 ++-- tox.ini | 38 ++----------------- 17 files changed, 75 insertions(+), 67 deletions(-) diff --git a/.travis.yml b/.travis.yml index 90c9640..8e2530d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,3 @@ -sudo: required -dist: trusty language: python branches: @@ -8,7 +6,9 @@ branches: python: - 2.7 - #- 3.4 + - 3.4 + - 3.5 + - 3.6 #- pypy #- pypy3 diff --git a/dev-requirements.txt b/dev-requirements.txt index 80fb301..31ae007 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -11,4 +11,4 @@ numpy>=1.8.2 # from trusty version # ros dependencies (necessary when running tests from install) -e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg --e git+https://github.com/asmodehn/genpy.git@setuptools#egg=ros_genpy +-e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic-py3#egg=ros_genpy diff --git a/pyros_msgs/typecheck/typechecker.py b/pyros_msgs/typecheck/typechecker.py index d795793..550dae3 100644 --- a/pyros_msgs/typecheck/typechecker.py +++ b/pyros_msgs/typecheck/typechecker.py @@ -39,7 +39,7 @@ The keys denote the message fields names, and the values are the types, following the previous rules. """ - +import sys import six # to get long for py2 and int for py3 six_long = six.integer_types[-1] @@ -83,7 +83,26 @@ def __call__(self, *args, **kwargs): #TODO : is it the same with slots ? elif args: # basic type (or mapping type but we pass a full value) # we sanitize by running the type initializer with the value - return self.t(*(a for a in args if a is not None)) # careful we do not want to pass None to a basic type + if self.t == six.binary_type: + # we need to encode + initializer = args[0] + # Ref : https://www.python.org/dev/peps/pep-0358/ + if isinstance(initializer, six.text_type) and sys.version_info > (3, 0): + encoding = args[1] if len(args) > 1 else sys.getdefaultencoding() + return self.t(initializer, encoding) + elif initializer is not None: + return self.t(initializer) + else: + return self.t() + elif self.t == six.text_type: + # we need to decode + obj = args[0] + if isinstance(args[0], six.binary_type): + obj.decode(sys.getdefaultencoding()) + return self.t(obj) + else: + return self.t( + *(a for a in args if a is not None)) # careful we do not want to pass None to a basic type else: TypeCheckerException("Calling {self} with {args} and {kwargs}. not supported".format(**locals())) @@ -191,22 +210,25 @@ class TypeCheckerException(Exception): def __init__(self, message): if isinstance(message, six.text_type): - super(TypeCheckerException, self).__init__(message.encode('utf-8')) + super(TypeCheckerException, self).__init__(message.encode('utf-8', 'replace')) self.message = message elif isinstance(message, six.binary_type): super(TypeCheckerException, self).__init__(message) - self.message = message.decode('utf-8') + self.message = message.decode('utf-8', 'replace') else: # This shouldn't happen... raise TypeError("message in an exception has to be a string (optionally unicode)") def __str__(self): # when we need a binary string - return self.message.encode('ascii', 'ignore') + return self.message.encode('ascii', 'replace') def __unicode__(self): # when we need a text string return self.message + def __repr__(self): + return "TypeCheckerException(" + self.message + ")" + class TypeChecker(object): def __init__(self, sanitizer, accepter): diff --git a/tests/test_pyros_msgs/opt_as_array/__init__.py b/tests/test_pyros_msgs/opt_as_array/__init__.py index 1ecdaa8..962feef 100644 --- a/tests/test_pyros_msgs/opt_as_array/__init__.py +++ b/tests/test_pyros_msgs/opt_as_array/__init__.py @@ -5,13 +5,8 @@ import site added_site_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'rosdeps') -print("Adding site directory {0} to access genmsg, genpy and, std_msgs".format(added_site_dir)) +print("Adding site directory {0} to access std_msgs".format(added_site_dir)) site.addsitedir(added_site_dir) -# Using genpy directly if ROS has been setup (while using from ROS pkg) -import genpy, genmsg - -print(sys.path) - import rosimport rosimport.activate() diff --git a/tests/test_pyros_msgs/opt_as_array/test_opt_string.py b/tests/test_pyros_msgs/opt_as_array/test_opt_string.py index 06bd1bb..cc096ba 100644 --- a/tests/test_pyros_msgs/opt_as_array/test_opt_string.py +++ b/tests/test_pyros_msgs/opt_as_array/test_opt_string.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, division, print_function +import sys from . import msg as test_gen_msgs @@ -20,7 +21,10 @@ ), max_size=1)) def test_init_rosdata(data): msg = test_gen_msgs.test_opt_string_as_array(data=data) - assert msg.data == data + if sys.version_info > (3, 0): # encoding/decoding is explicit + assert [d.decode(sys.getdefaultencoding()) for d in msg.data] == data + else: + assert msg.data == data @hypothesis.given(hypothesis.strategies.one_of( @@ -31,7 +35,10 @@ def test_init_rosdata(data): )) def test_init_data(data): msg = test_gen_msgs.test_opt_string_as_array(data=data) - assert msg.data == [data] + if sys.version_info > (3, 0): # encoding/decoding is explicit + assert msg.data[0].decode(sys.getdefaultencoding()) == data + else: + assert msg.data[0] == data @hypothesis.given(hypothesis.strategies.one_of( @@ -42,7 +49,10 @@ def test_init_data(data): )) def test_init_raw(data): msg = test_gen_msgs.test_opt_string_as_array(data) - assert msg.data == [data] + if sys.version_info > (3, 0): # encoding/decoding is explicit + assert msg.data[0].decode(sys.getdefaultencoding()) == data + else: + assert msg.data[0] == data def test_init_default(): diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_bool.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_bool.py index 7a5a204..cc69e3f 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_bool.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_bool.py @@ -44,7 +44,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_bool_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_int16.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_int16.py index 2b6cff2..73719bb 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_int16.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_int16.py @@ -51,7 +51,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_int16_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_int32.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_int32.py index 2523870..8e912de 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_int32.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_int32.py @@ -52,7 +52,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_int32_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_int64.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_int64.py index 0a1b734..907b066 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_int64.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_int64.py @@ -51,7 +51,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_int64_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_int8.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_int8.py index f25daca..a495f80 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_int8.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_int8.py @@ -52,7 +52,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_int8_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_string.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_string.py index e6c7369..4f7dac6 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_string.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_string.py @@ -1,6 +1,7 @@ from __future__ import absolute_import, division, print_function # TODO : check all types +import sys import pytest from . import msg as test_gen_msgs @@ -21,7 +22,10 @@ )) def test_init_rosdata(data): msg = test_gen_msgs.test_opt_string_as_nested(data=data) - assert msg.data == data + if sys.version_info > (3, 0): # encoding/decoding is explicit + assert msg.data.decode(sys.getdefaultencoding()) == data + else: + assert msg.data == data @hypothesis.given(hypothesis.strategies.one_of( @@ -32,7 +36,10 @@ def test_init_rosdata(data): )) def test_init_data(data): msg = test_gen_msgs.test_opt_string_as_nested(data=data) - assert msg.data == data + if sys.version_info > (3, 0): # encoding/decoding is explicit + assert msg.data.decode(sys.getdefaultencoding()) == data + else: + assert msg.data == data @hypothesis.given(hypothesis.strategies.one_of( @@ -43,7 +50,10 @@ def test_init_data(data): )) def test_init_raw(data): msg = test_gen_msgs.test_opt_string_as_nested(data) - assert msg.data == data + if sys.version_info > (3, 0): # encoding/decoding is explicit + assert msg.data.decode(sys.getdefaultencoding()) == data + else: + assert msg.data == data def test_init_default(): diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_uint16.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint16.py index baa701e..0890e45 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_uint16.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint16.py @@ -51,7 +51,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_uint16_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_uint32.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint32.py index 6f31884..898b2c9 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_uint32.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint32.py @@ -51,7 +51,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_uint32_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_uint64.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint64.py index 88f5e2b..df3af1a 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_uint64.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint64.py @@ -52,7 +52,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_uint64_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/tests/test_pyros_msgs/opt_as_nested/test_opt_uint8.py b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint8.py index 79ad93f..4592b47 100644 --- a/tests/test_pyros_msgs/opt_as_nested/test_opt_uint8.py +++ b/tests/test_pyros_msgs/opt_as_nested/test_opt_uint8.py @@ -51,7 +51,7 @@ def test_wrong_init_except(data): with pytest.raises(AttributeError) as cm: test_gen_msgs.test_opt_uint8_as_nested(data) assert isinstance(cm.value, AttributeError) - assert "does not match the accepted type schema for 'data' : Any of set" in cm.value.message + assert "does not match the accepted type schema for 'data' : Any of " in str(cm.value) # Just in case we run this directly if __name__ == '__main__': diff --git a/tests/test_pyros_msgs/typecheck/test_ros_mappings.py b/tests/test_pyros_msgs/typecheck/test_ros_mappings.py index a2b0085..64eb153 100644 --- a/tests/test_pyros_msgs/typecheck/test_ros_mappings.py +++ b/tests/test_pyros_msgs/typecheck/test_ros_mappings.py @@ -1,11 +1,12 @@ from __future__ import absolute_import, division, print_function import os +import six import sys import numpy import pytest import collections -from six import BytesIO +from six import BytesIO, StringIO import site # This is used for message definitions, not for python code @@ -128,10 +129,10 @@ def test_typechecker_serialize_deserialize_inverse(msg_type_and_ok_value): if isinstance(value, std_msgs.Float64): # for floats, this is only true relative to some epsilon... numpy.testing.assert_allclose(received.data, value.data) - if isinstance(value, std_msgs.Float32): # for floats, this is only true relative to some epsilon... + elif isinstance(value, std_msgs.Float32): # for floats, this is only true relative to some epsilon... numpy.testing.assert_allclose(received.data, value.data, rtol=1e-5) else: - assert received == value + assert received == msg_type_and_ok_value[1] @given(msg_type_and_value(std_msgs_types_strat_broken)) diff --git a/tox.ini b/tox.ini index 9af67ee..d3f7bbb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,15 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py27 -# ros distro does not matter in pure python tests -# envlist = py27-{indigo,jade,kinetic} -#, py34 +envlist = py27, py34, py35, py36 #, pypy #, pypy3 -# we skip sdist since currently our sdist doesnt generate messages -# we will manually setup the package via develop in [testenv] -#skipsdist=True - [travis] 2.7 = py27 - -#3.4 = py34 +3.4 = py34 +3.5 = py35 +3.6 = py36 # not tested yet #pypy = pypy @@ -35,30 +29,6 @@ deps = # to always force recreation and avoid unexpected side effects recreate=True -# to allow access to ROS packages -# sitepackages=True -# We do not want any access to system (same as basic travis python testing) - -;# TMP because our dependencies are not in pips -;usedevelop = True - -# we need to set our ROS distro in the pythonpath for our code to rely on ROS -#TODO : not havnig this breaks pyros-setup which tries to create its config file in -# pyros_setup.configurable_import().configure().activate() -#/usr/local/lib/python2.7/dist-packages/pyros_setup/__init__.py:92: in configure -# """) -#/usr/local/lib/python2.7/dist-packages/pyros_config/confighandler.py:74: in configure_file -# os.makedirs(os.path.dirname(cfg_filename)) -#.tox/py27-indigo/lib/python2.7/os.py:157: in makedirs -# mkdir(name, mode) -#E OSError: [Errno 13] Permission denied: '/usr/local/lib/python2.7/dist-packages/instance' - -setenv= - # TODO : get rid of this by removing all dependencies to ROS on system... -; indigo: PYTHONPATH=/opt/ros/indigo/lib/python2.7/dist-packages -; jade: PYTHONPATH=/opt/ros/jade/lib/python2.7/dist-packages -; kinetic: PYTHONPATH=/opt/ros/kinetic/lib/python2.7/dist-packages - commands= # we want to make sure python finds the installed package in tox env # and doesn't confuse with pyc generated during dev (which happens if we use self test feature here) From 501d1bc77850c3f19b56d03c9ea95db4b2fb2d1c Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 8 Aug 2017 15:53:50 +0900 Subject: [PATCH 26/28] configuring travis matrix with tox envs. --- .travis.yml | 19 ++++++++++-- requirements/indigo.txt | 14 +++++++++ .../kinetic.txt | 0 requirements/lunar.txt | 14 +++++++++ tox.ini | 31 ++++++++++++------- 5 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 requirements/indigo.txt rename dev-requirements.txt => requirements/kinetic.txt (100%) create mode 100644 requirements/lunar.txt diff --git a/.travis.yml b/.travis.yml index 8e2530d..0204f00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,14 +4,29 @@ branches: except: - gh-pages +env: + - ROS_DISTRO=indigo + - ROS_DISTRO=kinetic + # Not LTS + - ROS_DISTRO=lunar + python: + # always test python2 (default supported python version for ROS1) - 2.7 - - 3.4 - - 3.5 + # always test latest python3 (to guarantee recent python support) - 3.6 #- pypy #- pypy3 +# Add specific python3 versions +matrix: + include: + # explicitely matching python version to the version on the ubuntu distro supported by the ROS LTS distro + - python: 3.4 + env: ROS_DISTRO=indigo + - python: 3.5 + env: ROS_DISTRO=kinetic + before_install: install: diff --git a/requirements/indigo.txt b/requirements/indigo.txt new file mode 100644 index 0000000..80d09de --- /dev/null +++ b/requirements/indigo.txt @@ -0,0 +1,14 @@ +gitchangelog +twine +pytest>=2.8.0 # as per hypothesis requirement (careful with 2.5.1 on trusty) +pytest-xdist # for --boxed (careful with the version it will be moved out of xdist) +hypothesis>=3.0.1 # to target xenial LTS version +numpy>=1.8.2 # from trusty version + +# source access to latest filefinder and rosimport from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport + +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools-py3#egg=ros_genpy diff --git a/dev-requirements.txt b/requirements/kinetic.txt similarity index 100% rename from dev-requirements.txt rename to requirements/kinetic.txt diff --git a/requirements/lunar.txt b/requirements/lunar.txt new file mode 100644 index 0000000..31ae007 --- /dev/null +++ b/requirements/lunar.txt @@ -0,0 +1,14 @@ +gitchangelog +twine +pytest>=2.8.0 # as per hypothesis requirement (careful with 2.5.1 on trusty) +pytest-xdist # for --boxed (careful with the version it will be moved out of xdist) +hypothesis>=3.0.1 # to target xenial LTS version +numpy>=1.8.2 # from trusty version + +# source access to latest filefinder and rosimport from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport + +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic-py3#egg=ros_genpy diff --git a/tox.ini b/tox.ini index d3f7bbb..6afddde 100644 --- a/tox.ini +++ b/tox.ini @@ -1,30 +1,39 @@ # content of: tox.ini , put in same dir as setup.py [tox] -envlist = py27, py34, py35, py36 +# Only considering ROS LTS for simplicity +envlist = py27-{indigo,kinetic,lunar}, py34-indigo, py35-kinetic, py36-{indigo,kinetic,lunar} #, pypy #, pypy3 [travis] -2.7 = py27 -3.4 = py34 -3.5 = py35 -3.6 = py36 +python = + # we test every current ROS1 distro on python 2.7 (official python support for ROS1) + 2.7 : py27-{indigo, kinetic, lunar} + 3.4 : py34-indigo + 3.5 : py35-kinetic + # we test every current ROS1 distro on latest python (to ensure support from pure python) + 3.6 : py36-{indigo, kinetic, lunar} # not tested yet #pypy = pypy #pypy3 = pypy3 -# Note : We can depend on travis matrix if needed -;[travis:env] -;DJANGO = -; 1.7: django17 -; 1.8: django18, docs +# We depend on travis matrix +[travis:env] +ROS_DISTRO = + kinetic: kinetic + indigo: indigo + lunar: lunar [testenv] + +# Dependencies matching the version in each ROS distro deps = # TODO : check why / how install_requires are installed or not in tox environments... - -rdev-requirements.txt + indigo: -rrequirements/indigo.txt + kinetic: -rrequirements/kinetic.txt + lunar: -rrequirements/lunar.txt # to always force recreation and avoid unexpected side effects recreate=True From 39368135eb16bd3370050dc93d76d63b7ca9571d Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 8 Aug 2017 16:04:40 +0900 Subject: [PATCH 27/28] attempting to fix link between travis and tox. dropping py3 fix for genpy dep. --- requirements/indigo.txt | 2 +- requirements/kinetic.txt | 2 +- requirements/lunar.txt | 2 +- tox.ini | 23 ++++++++++++----------- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/requirements/indigo.txt b/requirements/indigo.txt index 80d09de..80fb301 100644 --- a/requirements/indigo.txt +++ b/requirements/indigo.txt @@ -11,4 +11,4 @@ numpy>=1.8.2 # from trusty version # ros dependencies (necessary when running tests from install) -e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg --e git+https://github.com/asmodehn/genpy.git@setuptools-py3#egg=ros_genpy +-e git+https://github.com/asmodehn/genpy.git@setuptools#egg=ros_genpy diff --git a/requirements/kinetic.txt b/requirements/kinetic.txt index 31ae007..25fd621 100644 --- a/requirements/kinetic.txt +++ b/requirements/kinetic.txt @@ -11,4 +11,4 @@ numpy>=1.8.2 # from trusty version # ros dependencies (necessary when running tests from install) -e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg --e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic-py3#egg=ros_genpy +-e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic#egg=ros_genpy diff --git a/requirements/lunar.txt b/requirements/lunar.txt index 31ae007..25fd621 100644 --- a/requirements/lunar.txt +++ b/requirements/lunar.txt @@ -11,4 +11,4 @@ numpy>=1.8.2 # from trusty version # ros dependencies (necessary when running tests from install) -e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg --e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic-py3#egg=ros_genpy +-e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic#egg=ros_genpy diff --git a/tox.ini b/tox.ini index 6afddde..54e309b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ # content of: tox.ini , put in same dir as setup.py [tox] # Only considering ROS LTS for simplicity -envlist = py27-{indigo,kinetic,lunar}, py34-indigo, py35-kinetic, py36-{indigo,kinetic,lunar} +envlist = py27-ros1{indigo,kinetic,lunar}, py34-ros1indigo, py35-ros1kinetic, py36-ros1{indigo,kinetic,lunar} #, pypy #, pypy3 @@ -9,11 +9,12 @@ envlist = py27-{indigo,kinetic,lunar}, py34-indigo, py35-kinetic, py36-{indigo,k [travis] python = # we test every current ROS1 distro on python 2.7 (official python support for ROS1) - 2.7 : py27-{indigo, kinetic, lunar} - 3.4 : py34-indigo - 3.5 : py35-kinetic + 2.7 : py27 + # specific old python supported natively on ubuntu/ROS LTS distro + 3.4 : py34 + 3.5 : py35 # we test every current ROS1 distro on latest python (to ensure support from pure python) - 3.6 : py36-{indigo, kinetic, lunar} + 3.6 : py36 # not tested yet #pypy = pypy @@ -22,18 +23,18 @@ python = # We depend on travis matrix [travis:env] ROS_DISTRO = - kinetic: kinetic - indigo: indigo - lunar: lunar + kinetic: ros1kinetic + indigo: ros1indigo + lunar: ros1lunar [testenv] # Dependencies matching the version in each ROS distro deps = # TODO : check why / how install_requires are installed or not in tox environments... - indigo: -rrequirements/indigo.txt - kinetic: -rrequirements/kinetic.txt - lunar: -rrequirements/lunar.txt + ros1indigo: -rrequirements/indigo.txt + ros1kinetic: -rrequirements/kinetic.txt + ros1lunar: -rrequirements/lunar.txt # to always force recreation and avoid unexpected side effects recreate=True From e2b894e27d46ed5f98eb4b6fca7eecfd58b498fb Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 8 Aug 2017 18:35:36 +0900 Subject: [PATCH 28/28] restructuring requirements for different tests with different versions... --- .travis.yml | 2 ++ requirements/indigo/debs_in_venv.txt | 15 +++++++++ requirements/kinetic/debs_in_venv.txt | 15 +++++++++ requirements/lunar/debs_in_venv.txt | 14 ++++++++ requirements/{ => python}/indigo.txt | 15 ++++----- requirements/{ => python}/kinetic.txt | 15 ++++----- requirements/python/latest.txt | 10 ++++++ requirements/{ => python}/lunar.txt | 16 ++++----- requirements/python/tests.txt | 5 +++ requirements/tools.txt | 3 ++ tox.ini | 48 +++++++++++++++++++++------ 11 files changed, 120 insertions(+), 38 deletions(-) create mode 100644 requirements/indigo/debs_in_venv.txt create mode 100644 requirements/kinetic/debs_in_venv.txt create mode 100644 requirements/lunar/debs_in_venv.txt rename requirements/{ => python}/indigo.txt (59%) rename requirements/{ => python}/kinetic.txt (60%) create mode 100644 requirements/python/latest.txt rename requirements/{ => python}/lunar.txt (60%) create mode 100644 requirements/python/tests.txt create mode 100644 requirements/tools.txt diff --git a/.travis.yml b/.travis.yml index 0204f00..dc344e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,8 @@ env: - ROS_DISTRO=kinetic # Not LTS - ROS_DISTRO=lunar + # to get latest dependencies (not released in a ROS distro yet) + - ROS_DISTRO=latest python: # always test python2 (default supported python version for ROS1) diff --git a/requirements/indigo/debs_in_venv.txt b/requirements/indigo/debs_in_venv.txt new file mode 100644 index 0000000..b949165 --- /dev/null +++ b/requirements/indigo/debs_in_venv.txt @@ -0,0 +1,15 @@ +# trusty packages versions to validate behavior with these versions for a potential ROS package for pyros-msgs +pytest==2.5.1 +pytest-xdist==1.8 # for --boxed +hypothesis==3.0.1 # backported to indigo as https://github.com/asmodehn/hypothesis-rosrelease +numpy>=1.8.1 + +# TODO : lock this on indigo +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools#egg=ros_genpy + +# source access to latest filefinder and rosimport from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport + diff --git a/requirements/kinetic/debs_in_venv.txt b/requirements/kinetic/debs_in_venv.txt new file mode 100644 index 0000000..6f222e0 --- /dev/null +++ b/requirements/kinetic/debs_in_venv.txt @@ -0,0 +1,15 @@ +# xenial packages versions to validate behavior with these versions for a potential ROS package for pyros-msgs +pytest==2.8.7 +pytest-xdist==1.8 # for --boxed +hypothesis==3.0.1 +numpy==1.11.0 + +# TODO : lock this on kinetic +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic#egg=ros_genpy + +# source access to latest filefinder and rosimport from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport + diff --git a/requirements/lunar/debs_in_venv.txt b/requirements/lunar/debs_in_venv.txt new file mode 100644 index 0000000..e5c0fee --- /dev/null +++ b/requirements/lunar/debs_in_venv.txt @@ -0,0 +1,14 @@ +# xenial packages versions to validate behavior with these versions for a potential ROS package for pyros-msgs +pytest==2.8.7 +pytest-xdist==1.8 # for --boxed +hypothesis==3.0.1 +numpy==1.11.0 + +# TODO : lock this on lunar +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic#egg=ros_genpy + +# source access to latest filefinder and rosimport from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport diff --git a/requirements/indigo.txt b/requirements/python/indigo.txt similarity index 59% rename from requirements/indigo.txt rename to requirements/python/indigo.txt index 80fb301..3f95607 100644 --- a/requirements/indigo.txt +++ b/requirements/python/indigo.txt @@ -1,14 +1,11 @@ -gitchangelog -twine -pytest>=2.8.0 # as per hypothesis requirement (careful with 2.5.1 on trusty) -pytest-xdist # for --boxed (careful with the version it will be moved out of xdist) -hypothesis>=3.0.1 # to target xenial LTS version -numpy>=1.8.2 # from trusty version +# Requirements for running a pure python setup on top of indigo + +# TODO : lock this on indigo +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools#egg=ros_genpy # source access to latest filefinder and rosimport from git ... -e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 -e git+https://github.com/asmodehn/rosimport.git#egg=rosimport -# ros dependencies (necessary when running tests from install) --e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg --e git+https://github.com/asmodehn/genpy.git@setuptools#egg=ros_genpy diff --git a/requirements/kinetic.txt b/requirements/python/kinetic.txt similarity index 60% rename from requirements/kinetic.txt rename to requirements/python/kinetic.txt index 25fd621..8d456af 100644 --- a/requirements/kinetic.txt +++ b/requirements/python/kinetic.txt @@ -1,14 +1,11 @@ -gitchangelog -twine -pytest>=2.8.0 # as per hypothesis requirement (careful with 2.5.1 on trusty) -pytest-xdist # for --boxed (careful with the version it will be moved out of xdist) -hypothesis>=3.0.1 # to target xenial LTS version -numpy>=1.8.2 # from trusty version +# Requirements for running a pure python setup on top of kinetic + +# TODO : lock this on kinetic +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic#egg=ros_genpy # source access to latest filefinder and rosimport from git ... -e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 -e git+https://github.com/asmodehn/rosimport.git#egg=rosimport -# ros dependencies (necessary when running tests from install) --e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg --e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic#egg=ros_genpy diff --git a/requirements/python/latest.txt b/requirements/python/latest.txt new file mode 100644 index 0000000..8fbfdf0 --- /dev/null +++ b/requirements/python/latest.txt @@ -0,0 +1,10 @@ +# Requirements for running a pure python setup on top of the next upcoming rosdistro + +# TODO : lock this on lunar +# ros dependencies (necessary when running tests from install) +-e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg +-e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic#egg=ros_genpy + +# source access to latest filefinder and rosimport from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport diff --git a/requirements/lunar.txt b/requirements/python/lunar.txt similarity index 60% rename from requirements/lunar.txt rename to requirements/python/lunar.txt index 25fd621..4827006 100644 --- a/requirements/lunar.txt +++ b/requirements/python/lunar.txt @@ -1,14 +1,10 @@ -gitchangelog -twine -pytest>=2.8.0 # as per hypothesis requirement (careful with 2.5.1 on trusty) -pytest-xdist # for --boxed (careful with the version it will be moved out of xdist) -hypothesis>=3.0.1 # to target xenial LTS version -numpy>=1.8.2 # from trusty version - -# source access to latest filefinder and rosimport from git ... --e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 --e git+https://github.com/asmodehn/rosimport.git#egg=rosimport +# Requirements for running a pure python setup on top of lunar +# TODO : lock this on lunar # ros dependencies (necessary when running tests from install) -e git+https://github.com/asmodehn/genmsg.git@setuptools#egg=ros_genmsg -e git+https://github.com/asmodehn/genpy.git@setuptools-kinetic#egg=ros_genpy + +# source access to latest filefinder and rosimport from git ... +-e git+https://github.com/asmodehn/filefinder2.git#egg=filefinder2 +-e git+https://github.com/asmodehn/rosimport.git#egg=rosimport diff --git a/requirements/python/tests.txt b/requirements/python/tests.txt new file mode 100644 index 0000000..3ecc996 --- /dev/null +++ b/requirements/python/tests.txt @@ -0,0 +1,5 @@ +# latest version to validate behavior on pure python env +pytest +pytest-xdist # for --boxed +hypothesis +numpy diff --git a/requirements/tools.txt b/requirements/tools.txt new file mode 100644 index 0000000..7a9dbd5 --- /dev/null +++ b/requirements/tools.txt @@ -0,0 +1,3 @@ +# for release, version doesnt matter, we always work in virtualenv +gitchangelog +twine \ No newline at end of file diff --git a/tox.ini b/tox.ini index 54e309b..61034c5 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,22 @@ -# content of: tox.ini , put in same dir as setup.py +# tox.ini , put in same dir as setup.py [tox] -# Only considering ROS LTS for simplicity -envlist = py27-ros1{indigo,kinetic,lunar}, py34-ros1indigo, py35-ros1kinetic, py36-ros1{indigo,kinetic,lunar} + +envlist = + + # based on ros distro with ubuntu debs base + py27-debs_{indigo,kinetic,lunar}, + + # based on ros distro with python2 base + py27-py_{indigo,kinetic,lunar,latest}, + + py34-debs_indigo, + py35-debs_kinetic, + + # based on ros distro with ubuntu debs base + py36-debs_{indigo,kinetic,lunar}, + + # based on ros distro with python3 base + py36-py_{indigo,kinetic,lunar,latest} #, pypy #, pypy3 @@ -13,7 +28,7 @@ python = # specific old python supported natively on ubuntu/ROS LTS distro 3.4 : py34 3.5 : py35 - # we test every current ROS1 distro on latest python (to ensure support from pure python) + # we test every current ROS1 distro on latest python (to ensure support from latest python) 3.6 : py36 # not tested yet @@ -23,18 +38,31 @@ python = # We depend on travis matrix [travis:env] ROS_DISTRO = - kinetic: ros1kinetic - indigo: ros1indigo - lunar: ros1lunar + kinetic: debs_kinetic, py_kinetic + indigo: debs_indigo, py_indigo + lunar: debs_lunar, py_lunar + latest: py_latest [testenv] # Dependencies matching the version in each ROS distro deps = # TODO : check why / how install_requires are installed or not in tox environments... - ros1indigo: -rrequirements/indigo.txt - ros1kinetic: -rrequirements/kinetic.txt - ros1lunar: -rrequirements/lunar.txt + debs_indigo: -rrequirements/indigo/debs_in_venv.txt + debs_kinetic: -rrequirements/kinetic/debs_in_venv.txt + debs_lunar: -rrequirements/lunar/debs_in_venv.txt + + py_indigo: -rrequirements/python/indigo.txt + py_indigo: -rrequirements/python/tests.txt + + py_kinetic: -rrequirements/python/kinetic.txt + py_kinetic: -rrequirements/python/tests.txt + + py_lunar: -rrequirements/python/lunar.txt + py_lunar: -rrequirements/python/tests.txt + + py_latest: -rrequirements/python/latest.txt + py_latest: -rrequirements/python/tests.txt # to always force recreation and avoid unexpected side effects recreate=True