From b28a219867101b1f9342f49326409469d99a30ef Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 20 Jan 2017 14:24:50 +0900 Subject: [PATCH 01/24] adding cookiecutter pylibrary for testing --- test/pylibrary/.gitignore | 1 + test/pylibrary/CMakeLists.txt | 28 ++++++++++++++++++++++++++++ test/pylibrary/catkin_pip | 1 + test/pylibrary/package.xml | 17 +++++++++++++++++ 4 files changed, 47 insertions(+) create mode 100644 test/pylibrary/.gitignore create mode 100644 test/pylibrary/CMakeLists.txt create mode 120000 test/pylibrary/catkin_pip create mode 100644 test/pylibrary/package.xml diff --git a/test/pylibrary/.gitignore b/test/pylibrary/.gitignore new file mode 100644 index 0000000..381c50c --- /dev/null +++ b/test/pylibrary/.gitignore @@ -0,0 +1 @@ +mypippkg diff --git a/test/pylibrary/CMakeLists.txt b/test/pylibrary/CMakeLists.txt new file mode 100644 index 0000000..1fd639a --- /dev/null +++ b/test/pylibrary/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 2.8.12) +project(python_nameless) + +# This is a full project CMakeLists (in case we call it independently for tests or so) +set (PIP_PROJECT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python-nameless) + +# NEEDED ONLY FOR TESTING INDEPENDENTLY : Setting up catkin-pip from source via symlink +# add_subdirectory(catkin_pip) + +find_package(catkin REQUIRED COMPONENTS catkin_pip) + +# no requirements file in this project template + +# This replace catkin_python_setup() +# For devel, this will install all your unsatisfied pip dependencies. +# Upon install, this will bark if some dependencies are not already satisfied (by ROS). +catkin_pip_setup(${PIP_PROJECT_DIR}) + +# CAREFUL : all projects for test here will share the same workspace. pip might have conflicts... + +# Here we plug back into usual catkin package build flow +catkin_package() + +## Unit tests +if (CATKIN_ENABLE_TESTING) + catkin_add_nosetests(${PIP_PROJECT_DIR}/tests) + catkin_add_pytests(${PIP_PROJECT_DIR}/tests) +endif() diff --git a/test/pylibrary/catkin_pip b/test/pylibrary/catkin_pip new file mode 120000 index 0000000..0c9c92c --- /dev/null +++ b/test/pylibrary/catkin_pip @@ -0,0 +1 @@ +../catkin_pip \ No newline at end of file diff --git a/test/pylibrary/package.xml b/test/pylibrary/package.xml new file mode 100644 index 0000000..778c988 --- /dev/null +++ b/test/pylibrary/package.xml @@ -0,0 +1,17 @@ + + python_nameless + 0.0.1 + + Enhanced cookiecutter template for Python libraries. + + AlexV + Ionel Cristian Mărieș + BSD + https://github.com/ionelmc/cookiecutter-pylibrary.git + + catkin + catkin_pip + + + + From 14a70125166fc9cd50401da5975978b84019b7a2 Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 20 Jan 2017 14:30:10 +0900 Subject: [PATCH 02/24] adding cookiecutter pypackage for tests --- test/pypackage/.gitignore | 1 + test/pypackage/CMakeLists.txt | 35 +++++++++++++++++++++++++++++++++++ test/pypackage/catkin_pip | 1 + test/pypackage/package.xml | 17 +++++++++++++++++ 4 files changed, 54 insertions(+) create mode 100644 test/pypackage/.gitignore create mode 100644 test/pypackage/CMakeLists.txt create mode 120000 test/pypackage/catkin_pip create mode 100644 test/pypackage/package.xml diff --git a/test/pypackage/.gitignore b/test/pypackage/.gitignore new file mode 100644 index 0000000..381c50c --- /dev/null +++ b/test/pypackage/.gitignore @@ -0,0 +1 @@ +mypippkg diff --git a/test/pypackage/CMakeLists.txt b/test/pypackage/CMakeLists.txt new file mode 100644 index 0000000..10a781a --- /dev/null +++ b/test/pypackage/CMakeLists.txt @@ -0,0 +1,35 @@ +cmake_minimum_required(VERSION 2.8.12) +project(python_boilerplate) + +# This is a full project CMakeLists (in case we call it independently for tests or so) +set (PIP_PROJECT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python_boilerplate) + +# NEEDED ONLY FOR TESTING INDEPENDENTLY : Setting up catkin-pip from source via symlink +add_subdirectory(catkin_pip) + +find_package(catkin REQUIRED COMPONENTS catkin_pip) + +# For development, we need to install the pip dependencies in the workspace being created +# Upon install, this will not do anything. All requirements should be satisfied by ROS dependencies system. +# => pip dependencies will work from source only +# => to release a pip dependency in a ROS system, you need to catkinize all dependencies. +catkin_pip_requirements(${PIP_PROJECT_DIR}/requirements_dev.txt --ignore-installed) +# Here we ignore already installed packages, to avoid conflicts with system installed packages (pygments for example) +# However this also means that existing packages in the workspace will not be used to satisfy requirements, +# and will be reinstalled everytime... + +# This replace catkin_python_setup() +# For devel, this will install all your unsatisfied pip dependencies. +# Upon install, this will bark if some dependencies are not already satisfied (by ROS). +catkin_pip_setup(${PIP_PROJECT_DIR}) + +# CAREFUL : all projects for test here will share the same workspace. pip might have conflicts... + +# Here we plug back into usual catkin package build flow +catkin_package() + +## Unit tests +if (CATKIN_ENABLE_TESTING) + catkin_add_nosetests(${PIP_PROJECT_DIR}/tests) + catkin_add_pytests(${PIP_PROJECT_DIR}/tests) +endif() diff --git a/test/pypackage/catkin_pip b/test/pypackage/catkin_pip new file mode 120000 index 0000000..0c9c92c --- /dev/null +++ b/test/pypackage/catkin_pip @@ -0,0 +1 @@ +../catkin_pip \ No newline at end of file diff --git a/test/pypackage/package.xml b/test/pypackage/package.xml new file mode 100644 index 0000000..6c9f260 --- /dev/null +++ b/test/pypackage/package.xml @@ -0,0 +1,17 @@ + + python_boilerplate + 0.0.1 + + Cookiecutter template for a Python package. + + AlexV + Audrey Roy Greenfeld + BSD + https://github.com/audreyr/cookiecutter-pypackage.git + + catkin + catkin_pip + + + + From bb21982f6c67bb5a4aac4de5088674e3c864d51e Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 20 Jan 2017 14:51:33 +0900 Subject: [PATCH 03/24] adding cookiecutter pypackage-minimal for tests --- test/pypackage-minimal/.gitignore | 1 + test/pypackage-minimal/CMakeLists.txt | 28 +++++++++++++++++++++++++++ test/pypackage-minimal/package.xml | 18 +++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 test/pypackage-minimal/.gitignore create mode 100644 test/pypackage-minimal/CMakeLists.txt create mode 100644 test/pypackage-minimal/package.xml diff --git a/test/pypackage-minimal/.gitignore b/test/pypackage-minimal/.gitignore new file mode 100644 index 0000000..381c50c --- /dev/null +++ b/test/pypackage-minimal/.gitignore @@ -0,0 +1 @@ +mypippkg diff --git a/test/pypackage-minimal/CMakeLists.txt b/test/pypackage-minimal/CMakeLists.txt new file mode 100644 index 0000000..98bd7d4 --- /dev/null +++ b/test/pypackage-minimal/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 2.8.12) +project(cookiecutter_pypackage_minimal) + +# This is a full project CMakeLists (in case we call it independently for tests or so) +set (PIP_PROJECT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/cookiecutter_pypackage_minimal) + +# NEEDED ONLY FOR TESTING INDEPENDENTLY : Setting up catkin-pip from source via symlink +add_subdirectory(catkin_pip) + +find_package(catkin REQUIRED COMPONENTS catkin_pip) + +# no requirements file here + +# This replace catkin_python_setup() +# For devel, this will install all your unsatisfied pip dependencies. +# Upon install, this will bark if some dependencies are not already satisfied (by ROS). +catkin_pip_setup(${PIP_PROJECT_DIR}) + +# CAREFUL : all projects for test here will share the same workspace. pip might have conflicts... + +# Here we plug back into usual catkin package build flow +catkin_package() + +## Unit tests +# if (CATKIN_ENABLE_TESTING) +# catkin_add_nosetests(${PIP_PROJECT_DIR}/tests) +# catkin_add_pytests(${PIP_PROJECT_DIR}/tests) +# endif() diff --git a/test/pypackage-minimal/package.xml b/test/pypackage-minimal/package.xml new file mode 100644 index 0000000..08a616c --- /dev/null +++ b/test/pypackage-minimal/package.xml @@ -0,0 +1,18 @@ + + cookiecutter_pypackage_minimal + 0.0.1 + + An opinionated, minimal cookiecutter template for Python packages + + AlexV + Louis Taylor + MIT + https://github.com/asmodehn/cookiecutter-pypackage-minimal.git + + + catkin + catkin_pip + + + + From bf18052f1e5a7a60f835d561823012b83270e97f Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 20 Jan 2017 17:56:28 +0900 Subject: [PATCH 04/24] improved test cases to check the content of easy-install.pth --- cmake/catkin-pip-base.req | 4 +- test/CMakeLists.txt | 78 +++++++++++++----- test/catkin_pip_nosetests/test_pythonpath.py | 80 ------------------- .../__init__.py | 0 test/catkin_pip_pytest/conftest.py | 50 ++++++++++++ test/catkin_pip_pytest/test_easyinstallpth.py | 21 +++++ test/catkin_pip_pytest/test_site.py | 13 +++ test/catkin_pip_pytest/test_syspath.py | 58 ++++++++++++++ test/pipproject/CMakeLists.txt | 3 +- test/pylibrary/.gitignore | 2 +- test/pylibrary/package.xml | 2 +- test/pypackage-minimal/.gitignore | 2 +- test/pypackage-minimal/catkin_pip | 1 + test/pypackage/.gitignore | 2 +- test/pypackage/CMakeLists.txt | 2 +- 15 files changed, 208 insertions(+), 110 deletions(-) delete mode 100644 test/catkin_pip_nosetests/test_pythonpath.py rename test/{catkin_pip_nosetests => catkin_pip_pytest}/__init__.py (100%) create mode 100644 test/catkin_pip_pytest/conftest.py create mode 100644 test/catkin_pip_pytest/test_easyinstallpth.py create mode 100644 test/catkin_pip_pytest/test_site.py create mode 100644 test/catkin_pip_pytest/test_syspath.py create mode 120000 test/pypackage-minimal/catkin_pip diff --git a/cmake/catkin-pip-base.req b/cmake/catkin-pip-base.req index 03eb91f..5656ef4 100644 --- a/cmake/catkin-pip-base.req +++ b/cmake/catkin-pip-base.req @@ -2,5 +2,5 @@ # It is better here to pin packages to a verified version, especially since we are explointing a borderline usecase # However this means we need a short maintenance cycle to not be outdated by python folks... -pip==8.1.2 # pip > 9.0 tries to remove egg-info/entry_points.txt from /opt/yujin/amd64/indigo-devel/lib/python2.7/dist-packages/pyros-... even if we specify /home/alexv/ROS/gopher_bootstrap/build/catkin_pip_env/bin/pip install -e /home/alexv/ROS/gopher_bootstrap/src/pyros --ignore-installed --no-dependencies --prefix /home/alexv/ROS/gopher_bootstrap/devel -setuptools==32.3.1 # we hit a bug in 33.0.0 on undefined setuptools.Command... +pip==9.0.1 +setuptools==33.1.1 diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 62141af..167862e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -31,42 +31,76 @@ else() message(STATUS "cookiecutter found at ${CATKIN_COOKIECUTTER}") endif() -message(STATUS " ... Getting cookiecutter-pipproject template from cookiecutter ...") -execute_process( - COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/wdm0006/cookiecutter-pipproject.git - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pipproject - RESULT_VARIABLE PIP_RESULT - OUTPUT_VARIABLE PIP_VARIABLE -) -message(STATUS " ... Done ...") +if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pipproject/mypippkg") + message(STATUS " ... Getting cookiecutter-pipproject template from cookiecutter ...") + execute_process( + COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/wdm0006/cookiecutter-pipproject.git + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pipproject + RESULT_VARIABLE PIP_RESULT + OUTPUT_VARIABLE PIP_VARIABLE + ) + message(STATUS " ... Done ...") +endif() -# -# Configuring catkin-pip extras from source by ourselves -# we should reproduce catkin behavior here -# -# set (DEVELSPACE True) -# set (INSTALLSPACE False) -# configure_file( ../cmake/catkin-pip-setup.cmake.in ${CATKIN_DEVEL_PREFIX}/share/catkin_pip/cmake/catkin-pip-setup.cmake @ONLY) +if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pylibrary/python-nameless") + message(STATUS " ... Getting cookiecutter-pylibrary template from cookiecutter ...") + execute_process( + COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/ionelmc/cookiecutter-pylibrary + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pylibrary + RESULT_VARIABLE PIP_RESULT + OUTPUT_VARIABLE PIP_VARIABLE + ) + message(STATUS " ... Done ...") +endif() -# set (DEVELSPACE False) -# set (INSTALLSPACE True) -# configure_file( ../cmake/catkin-pip-setup.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/catkin_generated/installspace/catkin-pip-setup.cmake @ONLY) +if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pypackage/python_boilerplate") + message(STATUS " ... Getting cookiecutter-pypackage template from cookiecutter ...") + execute_process( + COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/audreyr/cookiecutter-pypackage + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pypackage + RESULT_VARIABLE PIP_RESULT + OUTPUT_VARIABLE PIP_VARIABLE + ) + message(STATUS " ... Done ...") +endif() + +if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pypackage-minimal/cookiecutter_pypackage_minimal") + message(STATUS " ... Getting cookiecutter-pypackage-minimal template from cookiecutter ...") + execute_process( + # TMP : fork to get proper default package name. upstream : https://github.com/kragniz/cookiecutter-pypackage-minimal + COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/asmodehn/cookiecutter-pypackage-minimal + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pypackage-minimal + RESULT_VARIABLE PIP_RESULT + OUTPUT_VARIABLE PIP_VARIABLE + ) + message(STATUS " ... Done ...") +endif() # # and make sure they still work with catkin by building them all as subprojects # +# Testing project itself +add_subdirectory(pipproject) + +# Testing project itself +add_subdirectory(pylibrary) + +# Testing project itself +add_subdirectory(pypackage) + +# Testing project itself +add_subdirectory(pypackage-minimal) + + if (CATKIN_ENABLE_TESTING) ######################## # Nose Tests should work ######################## - # Testing project itself - add_subdirectory(pipproject) - # Testing environment setup by catkin_pip - catkin_add_nosetests(catkin_pip_nosetests) + catkin_add_pytests(catkin_pip_pytest --build-dir ${CMAKE_BINARY_DIR}) endif() \ No newline at end of file diff --git a/test/catkin_pip_nosetests/test_pythonpath.py b/test/catkin_pip_nosetests/test_pythonpath.py deleted file mode 100644 index bce792c..0000000 --- a/test/catkin_pip_nosetests/test_pythonpath.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import absolute_import -from __future__ import print_function - -import os -import sys -import site - - -def test_site(): - for p in site.getsitepackages(): - assert p in sys.path, "site packages directory {p} not found in {sys.path}".format(**locals()) - - -def test_extra_paths(): - #print(__file__) - mod_path_split = os.path.abspath(__file__).split(os.sep) - # file should be absolute : we replace the first term - if mod_path_split[0] == '': - mod_path_split[0] = os.sep - #print(mod_path_split) - - last_idx = len(mod_path_split) - 2 - #print(mod_path_split[:-last_idx]) - - # lets find the git working tree - while not os.path.exists(os.path.join(*(mod_path_split[:-last_idx] + ['.git']))) and last_idx > 0: - last_idx -= 1 - - assert last_idx > 0, "No git working tree found while walking up to {0}".format(os.path.join(*mod_path_split)) - gs = os.path.join(*(mod_path_split[:-last_idx])) - # expected the testbuild folder (from travis check script) - assert os.path.exists(os.path.join(gs, 'testbuild')), "testbuild folder not found in {0}".format(gs) - - assert os.path.exists(os.path.join(gs, 'testbuild', 'devel', 'setup.bash')), "devel/setup.bash not found in {0}".format(gs) - ws = os.path.join(gs, 'testbuild', 'devel') - #print(ws) - - print(sys.path) - - # Verifying relative path order - if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - assert '/opt/ros/indigo/lib/python2.7/dist-packages' in sys.path, "{p} not in {sp}".format(p='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) - # TODO : It seems env_cached.sh append dist-packages path at the end of python_path somehow... We should investigate. - # HINT : only sourcing /opt/ros/indigo/setup.bash seems fine (dist-packages added at beginning of sys.path) - # HINT : env_cached.sh seems fine... maybe it comes from the way nosetests/py.test is launched ? - # HINT : following same process (building and running tests) with rospy_tutorials doesnt show the broken behavior. Something in catkin_pip breaking it ? - #assert len(sys.path) == len(set(sys.path)) # check unicity of paths - - if os.path.exists(os.path.join(ws, 'lib/python2.7/dist-packages')): - assert os.path.join(ws, 'lib/python2.7/dist-packages') in sys.path, "{p} not in {sp}".format(p=os.path.join(ws, 'lib/python2.7/dist-packages'), sp=sys.path) - if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - assert sys.path.index(os.path.join(ws, 'lib/python2.7/dist-packages')) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2} in {sp}".format(p1=os.path.join(ws, 'lib/python2.7/dist-packages'), p2='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) - - if os.path.exists(os.path.join(ws, 'lib/python2.7/site-packages')): - assert os.path.join(ws, 'lib/python2.7/site-packages') in sys.path, "{p} not in {sp}".format(p=os.path.join(ws, 'lib/python2.7/site-packages'), sp=sys.path) - - if os.path.exists(os.path.join(ws, 'lib/python2.7/dist-packages')): - assert sys.path.index(os.path.join(ws, 'lib/python2.7/site-packages')) < sys.path.index(os.path.join(ws, 'lib/python2.7/dist-packages')), "{p1} not before {p2} in {sp}".format(p1=os.path.join(ws, 'lib/python2.7/site-packages'), p2=os.path.join(ws, 'lib/python2.7/dist-packages'), sp=sys.path) - - if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - assert sys.path.index(os.path.join(ws, 'lib/python2.7/site-packages')) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2} in {sp}".format(p1=os.path.join(ws, 'lib/python2.7/site-packages'), p2='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) - - # Verifying the pip editable installed package location is in python path - ss = os.path.join(gs, 'test', 'pipproject', 'mypippkg') - - if os.path.exists(ss): - assert ss in sys.path, "{p} not in {sp}".format(p=ss, sp=sys.path) - - # Lets keep it simple and rely on python way of handling easy-install.pth and egg-link before we modify it... - # if os.path.exists(os.path.join(ws, 'lib/python2.7/site-packages')): - # assert sys.path.index(ss) < sys.path.index(os.path.join(ws, 'lib/python2.7/site-packages')), "{p1} not before {p2} in {sp}".format(p1=ss, p2=os.path.join(ws, 'lib/python2.7/site-packages'), sp=sys.path) - # - # if os.path.exists(os.path.join(ws, 'lib/python2.7/dist-packages')): - # assert sys.path.index(ss) < sys.path.index(os.path.join(ws, 'lib/python2.7/dist-packages')), "{p1} not before {p2} in {sp}".format(p1=ss, p2=os.path.join(ws, 'lib/python2.7/dist-packages'), sp=sys.path) - # - # if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - # assert sys.path.index(ss) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2} in {sp}".format(p1=ss, p2='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) - - - #TODO : verify the position of the catkin_pip_env paths (bin? and site-packages) diff --git a/test/catkin_pip_nosetests/__init__.py b/test/catkin_pip_pytest/__init__.py similarity index 100% rename from test/catkin_pip_nosetests/__init__.py rename to test/catkin_pip_pytest/__init__.py diff --git a/test/catkin_pip_pytest/conftest.py b/test/catkin_pip_pytest/conftest.py new file mode 100644 index 0000000..11ad3ef --- /dev/null +++ b/test/catkin_pip_pytest/conftest.py @@ -0,0 +1,50 @@ +import os +import pytest + + +def pytest_addoption(parser): + parser.addoption("--build-dir", action="store", default="build", help="the build directory") + + +@pytest.fixture +def build_dir(request): + return request.config.getoption("--build-dir") + + +@pytest.fixture +def git_working_tree(request): + + mod_path_split = os.path.abspath(__file__).split(os.sep) + # file should be absolute : we replace the first term + if mod_path_split[0] == '': + mod_path_split[0] = os.sep + #print(mod_path_split) + + last_idx = len(mod_path_split) - 2 + #print(mod_path_split[:-last_idx]) + + # lets find the git working tree + while not os.path.exists(os.path.join(*(mod_path_split[:-last_idx] + ['.git']))) and last_idx > 0: + last_idx -= 1 + + assert last_idx > 0, "No git working tree found while walking up to {0}".format(os.path.join(*mod_path_split)) + gs = os.path.join(*(mod_path_split[:-last_idx])) + + return gs + + +@pytest.fixture +def devel_space(request): + # lets find devel space in current dir (cwd should be the build directory) + # expected setup.sh + assert os.path.exists(os.path.abspath(os.path.join('devel', 'setup.sh'))), "devel/setup.bash not found in {0}".format(os.getcwd()) + ws = os.path.abspath('devel') + return ws + + +@pytest.fixture +def build_dir(request): + # current dir should be the build directory (binary dir for cmake) + # otherwise cmake tests commands are broken + return os.getcwd() + diff --git a/test/catkin_pip_pytest/test_easyinstallpth.py b/test/catkin_pip_pytest/test_easyinstallpth.py new file mode 100644 index 0000000..cbecdd5 --- /dev/null +++ b/test/catkin_pip_pytest/test_easyinstallpth.py @@ -0,0 +1,21 @@ +from __future__ import absolute_import +from __future__ import print_function + +import os + + +def test_easyinstall_content(devel_space, git_working_tree): + easy_install_pth_path = os.path.join(devel_space, 'lib','python2.7','site-packages','easy-install.pth') + assert os.path.exists(easy_install_pth_path) + with open(easy_install_pth_path) as easy_install_pth: + editable_paths = easy_install_pth.read().splitlines() + print(editable_paths) + for python_pkg in [ + os.path.join('pipproject', 'mypippkg'), + os.path.join('pylibrary', 'python-nameless', 'src'), + os.path.join('pypackage', 'python_boilerplate'), + os.path.join('pypackage-minimal', 'cookiecutter_pypackage_minimal'), + ]: + assert os.path.join(git_working_tree, 'test', python_pkg) in editable_paths, "{0} not in easy-install.pth".format(os.path.join(git_working_tree, 'test', python_pkg)) + + diff --git a/test/catkin_pip_pytest/test_site.py b/test/catkin_pip_pytest/test_site.py new file mode 100644 index 0000000..84cff57 --- /dev/null +++ b/test/catkin_pip_pytest/test_site.py @@ -0,0 +1,13 @@ +from __future__ import absolute_import +from __future__ import print_function + +import os +import sys +import site + + +# TODO : is this really useful ? +# we are not using any feature of site module itself (currently at least, and API is quite limited...) +def test_site(): + for p in site.getsitepackages(): + assert p in sys.path, "site packages directory {p} not found in {sys.path}".format(**locals()) diff --git a/test/catkin_pip_pytest/test_syspath.py b/test/catkin_pip_pytest/test_syspath.py new file mode 100644 index 0000000..88e17e4 --- /dev/null +++ b/test/catkin_pip_pytest/test_syspath.py @@ -0,0 +1,58 @@ +from __future__ import absolute_import +from __future__ import print_function + +import os +import sys +import site + + +def test_sys_path(devel_space): + print(sys.path) + + # Verifying relative path order + if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): + assert '/opt/ros/indigo/lib/python2.7/dist-packages' in sys.path, "{p} not in {sp}".format(p='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) + # TODO : It seems env_cached.sh append dist-packages path at the end of python_path somehow... We should investigate. + # HINT : only sourcing /opt/ros/indigo/setup.bash seems fine (dist-packages added at beginning of sys.path) + # HINT : env_cached.sh seems fine... maybe it comes from the way nosetests/py.test is launched ? + # HINT : following same process (building and running tests) with rospy_tutorials doesnt show the broken behavior. Something in catkin_pip breaking it ? + #assert len(sys.path) == len(set(sys.path)) # check unicity of paths + + if os.path.exists(os.path.join(devel_space, 'lib/python2.7/dist-packages')): + assert os.path.join(devel_space, 'lib/python2.7/dist-packages') in sys.path, "{p} not in {sp}".format(p=os.path.join(devel_space, 'lib/python2.7/dist-packages'), sp=sys.path) + if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): + assert sys.path.index(os.path.join(devel_space, 'lib/python2.7/dist-packages')) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2} in {sp}".format(p1=os.path.join(devel_space, 'lib/python2.7/dist-packages'), p2='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) + + if os.path.exists(os.path.join(devel_space, 'lib/python2.7/site-packages')): + assert os.path.join(devel_space, 'lib/python2.7/site-packages') in sys.path, "{p} not in {sp}".format(p=os.path.join(devel_space, 'lib/python2.7/site-packages'), sp=sys.path) + + if os.path.exists(os.path.join(devel_space, 'lib/python2.7/dist-packages')): + assert sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/dist-packages')), "{p1} not before {p2} in {sp}".format(p1=os.path.join(devel_space, 'lib/python2.7/site-packages'), p2=os.path.join(devel_space, 'lib/python2.7/dist-packages'), sp=sys.path) + + if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): + assert sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2} in {sp}".format(p1=os.path.join(devel_space, 'lib/python2.7/site-packages'), p2='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) + + +def test_sys_path_editable(git_working_tree, devel_space): + print(sys.path) + + # Verifying the pip editable installed package location is in python path + ss = os.path.join(git_working_tree, 'test', 'pipproject', 'mypippkg') + + if os.path.exists(ss): + assert ss in sys.path, "{p} not in {sp}".format(p=ss, sp=sys.path) + + # Lets keep it simple and rely on python way of handling easy-install.pth and egg-link before we modify it... + # It seems that by default the eggs path are aded after the pythonpaths... + + # if os.path.exists(os.path.join(devel_space, 'lib/python2.7/site-packages')): + # assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/site-packages')) + # + # if os.path.exists(os.path.join(devel_space, 'lib/python2.7/dist-packages')): + # assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/dist-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/dist-packages')) + # + # if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): + # assert sys.path.index(ss) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2}".format(p1=ss, p2='/opt/ros/indigo/lib/python2.7/dist-packages') + + + #TODO : verify the position of the catkin_pip_env paths (bin? and site-packages) diff --git a/test/pipproject/CMakeLists.txt b/test/pipproject/CMakeLists.txt index 32ef48c..7c3ae2b 100644 --- a/test/pipproject/CMakeLists.txt +++ b/test/pipproject/CMakeLists.txt @@ -28,5 +28,6 @@ catkin_package() ## Unit tests if (CATKIN_ENABLE_TESTING) catkin_add_nosetests(${PIP_PROJECT_DIR}/tests) - catkin_add_pytests(${PIP_PROJECT_DIR}/tests) + # disabling pytest for now since it fails if there is no test detected (no tests implemented at the moment). + # catkin_add_pytests(${PIP_PROJECT_DIR}/tests) endif() diff --git a/test/pylibrary/.gitignore b/test/pylibrary/.gitignore index 381c50c..ce99d71 100644 --- a/test/pylibrary/.gitignore +++ b/test/pylibrary/.gitignore @@ -1 +1 @@ -mypippkg +python-nameless diff --git a/test/pylibrary/package.xml b/test/pylibrary/package.xml index 778c988..563c909 100644 --- a/test/pylibrary/package.xml +++ b/test/pylibrary/package.xml @@ -10,7 +10,7 @@ https://github.com/ionelmc/cookiecutter-pylibrary.git catkin - catkin_pip + catkin_pip diff --git a/test/pypackage-minimal/.gitignore b/test/pypackage-minimal/.gitignore index 381c50c..eb4a8fb 100644 --- a/test/pypackage-minimal/.gitignore +++ b/test/pypackage-minimal/.gitignore @@ -1 +1 @@ -mypippkg +cookiecutter_pypackage_minimal diff --git a/test/pypackage-minimal/catkin_pip b/test/pypackage-minimal/catkin_pip new file mode 120000 index 0000000..0c9c92c --- /dev/null +++ b/test/pypackage-minimal/catkin_pip @@ -0,0 +1 @@ +../catkin_pip \ No newline at end of file diff --git a/test/pypackage/.gitignore b/test/pypackage/.gitignore index 381c50c..ecf8385 100644 --- a/test/pypackage/.gitignore +++ b/test/pypackage/.gitignore @@ -1 +1 @@ -mypippkg +python_boilerplate diff --git a/test/pypackage/CMakeLists.txt b/test/pypackage/CMakeLists.txt index 10a781a..a912f87 100644 --- a/test/pypackage/CMakeLists.txt +++ b/test/pypackage/CMakeLists.txt @@ -5,7 +5,7 @@ project(python_boilerplate) set (PIP_PROJECT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python_boilerplate) # NEEDED ONLY FOR TESTING INDEPENDENTLY : Setting up catkin-pip from source via symlink -add_subdirectory(catkin_pip) +# add_subdirectory(catkin_pip) find_package(catkin REQUIRED COMPONENTS catkin_pip) From 7a027ae69e514177f08818cd5e734e21ee6fe98c Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 20 Jan 2017 18:02:38 +0900 Subject: [PATCH 05/24] added comment about sourcing install/setup.bash --- cmake/env-hooks/42.site_packages.sh.installspace.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/env-hooks/42.site_packages.sh.installspace.in b/cmake/env-hooks/42.site_packages.sh.installspace.in index c26fa3f..3cc240b 100755 --- a/cmake/env-hooks/42.site_packages.sh.installspace.in +++ b/cmake/env-hooks/42.site_packages.sh.installspace.in @@ -6,6 +6,8 @@ # In install space we only need the catkin_pip environment setup. # Install space should behave like packages, everything in debian layout. +# CAREFUL : sourcing install.setup.bash will also source devel/setup.bash if folder exists... + # Setting up the catkin environment itself # This needs to be in a different prefix, to be able to generate an install dir with setuptools, yet clear of any catkin_pip artifacts. PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") From 2e4ec5a5b4848a3e882fcd012e0c3b09695a936d Mon Sep 17 00:00:00 2001 From: alexv Date: Fri, 20 Jan 2017 18:28:56 +0900 Subject: [PATCH 06/24] reverted to pip 8.1.2 changes added in prevision of switching to pip 9.X when --ignore-installed working... --- cmake/catkin-pip-base.req | 3 ++- cmake/catkin-pip-setup.cmake.in | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cmake/catkin-pip-base.req b/cmake/catkin-pip-base.req index 5656ef4..5a35c07 100644 --- a/cmake/catkin-pip-base.req +++ b/cmake/catkin-pip-base.req @@ -2,5 +2,6 @@ # It is better here to pin packages to a verified version, especially since we are explointing a borderline usecase # However this means we need a short maintenance cycle to not be outdated by python folks... -pip==9.0.1 +pip==8.1.2 # pip > 9.0 tries to remove system packages data, even if we specify /home/alexv/ROS/gopher_bootstrap/build/catkin_pip_env/bin/pip install -e /home/alexv/ROS/gopher_bootstrap/src/pyros --ignore-installed --no-dependencies --prefix /home/alexv/ROS/gopher_bootstrap/devel +# Issue : https://github.com/asmodehn/catkin_pip/issues/58 setuptools==33.1.1 diff --git a/cmake/catkin-pip-setup.cmake.in b/cmake/catkin-pip-setup.cmake.in index 88291ca..2702ab5 100644 --- a/cmake/catkin-pip-setup.cmake.in +++ b/cmake/catkin-pip-setup.cmake.in @@ -91,10 +91,12 @@ function(catkin_pip_setup) # Note : environment should already be setup at configure time for devel # Then we can run the pip command + # Note when installing in editable mode (for development) we shouldnt care about already installed versions. + # However : https://github.com/asmodehn/catkin_pip/issues/58 if(CATKIN_PIP_NO_DEPS) - catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --no-dependencies --prefix "${CATKIN_DEVEL_PREFIX}") + catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --no-dependencies --prefix "${CATKIN_DEVEL_PREFIX}" --ignore-installed) else() - catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --prefix "${CATKIN_DEVEL_PREFIX}") + catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --prefix "${CATKIN_DEVEL_PREFIX}" --ignore-installed) endif() if(NOT EXISTS ${package_path}/setup.py) From 3133fb5180417f409430229678868a0fd5bffdf3 Mon Sep 17 00:00:00 2001 From: alexv Date: Thu, 26 Jan 2017 16:47:43 +0900 Subject: [PATCH 07/24] API changed ! redesigned workflow by doing "pip install -e package" during make stage instead of configure. All tests passing. --- .gitignore | 1 + CMakeLists.txt | 2 +- ...p.cmake.in => catkin-pip-package.cmake.in} | 17 ++++++-- cmake/catkin-pip-runcmd.cmake.in | 40 ++++++++++++++++++- cmake/catkin-pip.cmake.in | 8 ++-- .../42.site_packages.sh.develspace.in | 14 +++---- test/CMakeLists.txt | 11 ++++- test/pipproject/CMakeLists.txt | 11 ++--- test/pylibrary/CMakeLists.txt | 11 ++--- test/pypackage-minimal/CMakeLists.txt | 11 ++--- test/pypackage/CMakeLists.txt | 11 ++--- 11 files changed, 91 insertions(+), 46 deletions(-) rename cmake/{catkin-pip-setup.cmake.in => catkin-pip-package.cmake.in} (81%) diff --git a/.gitignore b/.gitignore index 52728eb..d7102b8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ build testbuild test/mypippkg .idea +*.pyc \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f867a3..7b07a86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,7 @@ catkin_package( CFG_EXTRAS catkin-pip.cmake catkin-pip-runcmd.cmake - catkin-pip-setup.cmake + catkin-pip-package.cmake catkin-pip-prefix.cmake catkin-pip-requirements.cmake pytest.cmake diff --git a/cmake/catkin-pip-setup.cmake.in b/cmake/catkin-pip-package.cmake.in similarity index 81% rename from cmake/catkin-pip-setup.cmake.in rename to cmake/catkin-pip-package.cmake.in index 2702ab5..d027e4c 100644 --- a/cmake/catkin-pip-setup.cmake.in +++ b/cmake/catkin-pip-package.cmake.in @@ -1,4 +1,4 @@ -message(STATUS "Loading catkin-pip-setup.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") +message(STATUS "Loading catkin-pip-package.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) @@ -75,7 +75,10 @@ function(catkin_pip_python_setup) endfunction() -function(catkin_pip_setup) +# TODO : we now create an actual cmake target -> rename to catkin_pip_target ? +function(catkin_pip_package package_name) + + set(${PROJECT_NAME}_PIP_TARGET ${package_name} CACHE STRING "Make target generated to install this pip package as --editable for development") set (extra_macro_args ${ARGN}) @@ -94,15 +97,18 @@ function(catkin_pip_setup) # Note when installing in editable mode (for development) we shouldnt care about already installed versions. # However : https://github.com/asmodehn/catkin_pip/issues/58 if(CATKIN_PIP_NO_DEPS) - catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --no-dependencies --prefix "${CATKIN_DEVEL_PREFIX}" --ignore-installed) + catkin_pip_install_devel_target(${package_name} ${package_path} --no-dependencies --ignore-installed) + #catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --no-dependencies --prefix "${CATKIN_DEVEL_PREFIX}" --ignore-installed) else() - catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --prefix "${CATKIN_DEVEL_PREFIX}" --ignore-installed) + catkin_pip_install_devel_target(${package_name} ${package_path} --ignore-installed) + #catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --prefix "${CATKIN_DEVEL_PREFIX}" --ignore-installed) endif() if(NOT EXISTS ${package_path}/setup.py) message(FATAL_ERROR "catkin_pip_setup() called without 'setup.py' in project folder '${package_path}'") endif() + # Probably better to do that here (dont do anything "special" for python) catkin_pip_python_setup() # TODO : we might want to generate a package.xml on the fly from setup.py contents... @@ -110,4 +116,7 @@ function(catkin_pip_setup) # Hijacking catkin scripts again #set(_PACKAGE_XML_DIRECTORY ${package_path}) + # Here we plug back into usual catkin package build flow + catkin_package() + endfunction() diff --git a/cmake/catkin-pip-runcmd.cmake.in b/cmake/catkin-pip-runcmd.cmake.in index a910815..0181314 100644 --- a/cmake/catkin-pip-runcmd.cmake.in +++ b/cmake/catkin-pip-runcmd.cmake.in @@ -38,4 +38,42 @@ function(catkin_pip_runcmd) message(FATAL_ERROR "${PIP_ERROR}") endif() -endfunction() \ No newline at end of file +endfunction() + + +function(catkin_pip_install_devel_target package_name package_path) + set(CATKIN_PIP_INSTALL_DEVEL_OUTPUTS + # ${CATKIN_DEVEL_PREFIX}/${CATKIN_PIP_PYTHON_INSTALL_DIR}/easy-install.pth + # CAREFUL with multiple outputs : https://cmake.org/Bug/view.php?id=15116 + ${CATKIN_DEVEL_PREFIX}/${CATKIN_PIP_PYTHON_INSTALL_DIR}/${package_name}.egg-link + ) + + set(CATKIN_PIP_COMMAND ${CATKIN_PIP} install -e ${package_path} --prefix "${CATKIN_DEVEL_PREFIX}" ${ARGN}) + # TODO : change to use --target to allow installing duplicate packages... Ref https://github.com/pypa/pip/issues/4243 + + string(REPLACE ";" " " CATKIN_PIP_CMDSTR "${CATKIN_ENV} ${CATKIN_PIP_COMMAND}") + + add_custom_command( + OUTPUT ${CATKIN_PIP_INSTALL_DEVEL_OUTPUTS} + # Note we need to use catkin_env to make sure our envhook is properly loaded + COMMAND ${CATKIN_ENV} ${CATKIN_PIP_COMMAND} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DEPENDS "${package_path}" # file level dependency : package needs to be in the path + COMMENT "${CATKIN_PIP_CMDSTR}" + VERBATIM + ) + + add_custom_target(${package_name} DEPENDS ${CATKIN_PIP_INSTALL_DEVEL_OUTPUTS}) # file level dependency : we need the output of the custom command + + if (CATKIN_PIP_DEVEL_TARGETS) + add_dependencies(${package_name} ${CATKIN_PIP_DEVEL_TARGETS}) # target level dependency to prevent concurrency + endif() + + # To have all target install one after the other (even if user attempts concurrent run) + # since pip doesnt support concurrent runs + set( CATKIN_PIP_DEVEL_TARGETS + ${CATKIN_PIP_DEVEL_TARGETS} ${package_name} + CACHE INTERNAL "List of targets for pip for devel workspace" + ) + +endfunction() diff --git a/cmake/catkin-pip.cmake.in b/cmake/catkin-pip.cmake.in index b726b93..3940bbc 100644 --- a/cmake/catkin-pip.cmake.in +++ b/cmake/catkin-pip.cmake.in @@ -25,10 +25,10 @@ IF ( NOT CATKIN_PIP_REQUIREMENTS_FOUND ) ENDIF ( NOT CATKIN_PIP_REQUIREMENTS_FOUND ) # protecting against missing cmake file dependency -include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-setup.cmake" RESULT_VARIABLE CATKIN_PIP_SETUP_FOUND ) -IF ( NOT CATKIN_PIP_SETUP_FOUND ) - message ( FATAL_ERROR "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-setup.cmake Not Found !!!" ) -ENDIF ( NOT CATKIN_PIP_SETUP_FOUND ) +include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-package.cmake" RESULT_VARIABLE CATKIN_PIP_PACKAGE_FOUND ) +IF ( NOT CATKIN_PIP_PACKAGE_FOUND ) + message ( FATAL_ERROR "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-package.cmake Not Found !!!" ) +ENDIF ( NOT CATKIN_PIP_PACKAGE_FOUND ) # Setting our paths to package env-hooks provided by catkin-pip if ( NOT CATKIN_PIP_ENV_HOOKS_PATH ) diff --git a/cmake/env-hooks/42.site_packages.sh.develspace.in b/cmake/env-hooks/42.site_packages.sh.develspace.in index a9f5147..94e3b15 100755 --- a/cmake/env-hooks/42.site_packages.sh.develspace.in +++ b/cmake/env-hooks/42.site_packages.sh.develspace.in @@ -35,13 +35,13 @@ PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/ # Prepending here our easy-install.pth content into PYTHONPATH, for overlayed workspace to get it before the workspace path # Somehow Python pth/.egg behavior doesnt work well with workspace overlays, based on PYTHONPATH. -# TODO : This should be reenabled, unless we find the root cause and fix it -#if [ -f "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" ];then -# while read p; do -# #echo "Inserting $p before "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" into PYTHONPATH" -# PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "$p" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") -# done <"@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" -#fi +# However be aware of : https://github.com/pypa/pip/issues/3160 +if [ -f "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" ];then + while read p; do + #echo "Inserting $p before "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" into PYTHONPATH" + PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "$p" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") + done <"@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" +fi # Exporting is required to make sure we get the new value in children processes (catkin_make, test runs, etc.) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 167862e..9421815 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -101,6 +101,15 @@ if (CATKIN_ENABLE_TESTING) ######################## # Testing environment setup by catkin_pip - catkin_add_pytests(catkin_pip_pytest --build-dir ${CMAKE_BINARY_DIR}) + # We need to add all subprojects as dependencies for this test + catkin_add_pytests( + catkin_pip_pytest + --build-dir ${CMAKE_BINARY_DIR} + DEPENDENCIES + ${mypippkg_PIP_TARGET} # from pipproject + ${nameless_PIP_TARGET} # from pylibrary + ${python_boilerplate_PIP_TARGET} # from pypackage + ${cookiecutter_pypackage_minimal_PIP_TARGET} # from pypackage-minimal + ) endif() \ No newline at end of file diff --git a/test/pipproject/CMakeLists.txt b/test/pipproject/CMakeLists.txt index 7c3ae2b..a93cb97 100644 --- a/test/pipproject/CMakeLists.txt +++ b/test/pipproject/CMakeLists.txt @@ -15,19 +15,16 @@ catkin_pip_requirements(${PIP_PROJECT_DIR}/requirements.txt --ignore-installed) # However this also means that existing packages in the workspace will not be used to satisfy requirements, # and will be reinstalled everytime... -# This replace catkin_python_setup() +# This replace both catkin_python_setup() and catkin_package() # For devel, this will install all your unsatisfied pip dependencies. # Upon install, this will bark if some dependencies are not already satisfied (by ROS). -catkin_pip_setup(${PIP_PROJECT_DIR}) +catkin_pip_package(mypippkg ${PIP_PROJECT_DIR}) # CAREFUL : all projects for test here will share the same workspace. pip might have conflicts... -# Here we plug back into usual catkin package build flow -catkin_package() - ## Unit tests if (CATKIN_ENABLE_TESTING) - catkin_add_nosetests(${PIP_PROJECT_DIR}/tests) + catkin_add_nosetests(${PIP_PROJECT_DIR}/tests DEPENDENCIES mypippkg) # disabling pytest for now since it fails if there is no test detected (no tests implemented at the moment). - # catkin_add_pytests(${PIP_PROJECT_DIR}/tests) + # catkin_add_pytests(${PIP_PROJECT_DIR}/tests DEPENDENCIES mypippkg) endif() diff --git a/test/pylibrary/CMakeLists.txt b/test/pylibrary/CMakeLists.txt index 1fd639a..b9f5922 100644 --- a/test/pylibrary/CMakeLists.txt +++ b/test/pylibrary/CMakeLists.txt @@ -11,18 +11,15 @@ find_package(catkin REQUIRED COMPONENTS catkin_pip) # no requirements file in this project template -# This replace catkin_python_setup() +# This replace both catkin_python_setup() and catkin_package() # For devel, this will install all your unsatisfied pip dependencies. # Upon install, this will bark if some dependencies are not already satisfied (by ROS). -catkin_pip_setup(${PIP_PROJECT_DIR}) +catkin_pip_package(nameless ${PIP_PROJECT_DIR}) # CAREFUL : all projects for test here will share the same workspace. pip might have conflicts... -# Here we plug back into usual catkin package build flow -catkin_package() - ## Unit tests if (CATKIN_ENABLE_TESTING) - catkin_add_nosetests(${PIP_PROJECT_DIR}/tests) - catkin_add_pytests(${PIP_PROJECT_DIR}/tests) + catkin_add_nosetests(${PIP_PROJECT_DIR}/tests DEPENDENCIES nameless) + catkin_add_pytests(${PIP_PROJECT_DIR}/tests DEPENDENCIES nameless) endif() diff --git a/test/pypackage-minimal/CMakeLists.txt b/test/pypackage-minimal/CMakeLists.txt index 98bd7d4..756d9b6 100644 --- a/test/pypackage-minimal/CMakeLists.txt +++ b/test/pypackage-minimal/CMakeLists.txt @@ -11,18 +11,15 @@ find_package(catkin REQUIRED COMPONENTS catkin_pip) # no requirements file here -# This replace catkin_python_setup() +# This replace both catkin_python_setup() and catkin_package() # For devel, this will install all your unsatisfied pip dependencies. # Upon install, this will bark if some dependencies are not already satisfied (by ROS). -catkin_pip_setup(${PIP_PROJECT_DIR}) +catkin_pip_package(cookiecutter_pypackage_minimal ${PIP_PROJECT_DIR}) # CAREFUL : all projects for test here will share the same workspace. pip might have conflicts... -# Here we plug back into usual catkin package build flow -catkin_package() - ## Unit tests # if (CATKIN_ENABLE_TESTING) -# catkin_add_nosetests(${PIP_PROJECT_DIR}/tests) -# catkin_add_pytests(${PIP_PROJECT_DIR}/tests) +# catkin_add_nosetests(${PIP_PROJECT_DIR}/tests DEPENDENCIES cookiecutter_pypackage_minimal) +# catkin_add_pytests(${PIP_PROJECT_DIR}/tests DEPENDENCIES cookiecutter_pypackage_minimal) # endif() diff --git a/test/pypackage/CMakeLists.txt b/test/pypackage/CMakeLists.txt index a912f87..9a08c05 100644 --- a/test/pypackage/CMakeLists.txt +++ b/test/pypackage/CMakeLists.txt @@ -18,18 +18,15 @@ catkin_pip_requirements(${PIP_PROJECT_DIR}/requirements_dev.txt --ignore-install # However this also means that existing packages in the workspace will not be used to satisfy requirements, # and will be reinstalled everytime... -# This replace catkin_python_setup() +# This replace both catkin_python_setup() and catkin_package() # For devel, this will install all your unsatisfied pip dependencies. # Upon install, this will bark if some dependencies are not already satisfied (by ROS). -catkin_pip_setup(${PIP_PROJECT_DIR}) +catkin_pip_package(python_boilerplate ${PIP_PROJECT_DIR}) # CAREFUL : all projects for test here will share the same workspace. pip might have conflicts... -# Here we plug back into usual catkin package build flow -catkin_package() - ## Unit tests if (CATKIN_ENABLE_TESTING) - catkin_add_nosetests(${PIP_PROJECT_DIR}/tests) - catkin_add_pytests(${PIP_PROJECT_DIR}/tests) + catkin_add_nosetests(${PIP_PROJECT_DIR}/tests DEPENDENCIES python_boilerplate) + catkin_add_pytests(${PIP_PROJECT_DIR}/tests DEPENDENCIES python_boilerplate) endif() From eb3a5255ace13e0b26f166f9eb2f09d5346cecfa Mon Sep 17 00:00:00 2001 From: alexv Date: Thu, 26 Jan 2017 17:02:39 +0900 Subject: [PATCH 08/24] adding libffi-dev as build_depend for python_boilerplate --- test/pypackage/package.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/pypackage/package.xml b/test/pypackage/package.xml index 6c9f260..e88b1af 100644 --- a/test/pypackage/package.xml +++ b/test/pypackage/package.xml @@ -12,6 +12,9 @@ catkin catkin_pip + + libffi-dev + From be6b6dc4f5ac8f12864595e6faa8b986f47de282 Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 31 Jan 2017 16:26:49 +0900 Subject: [PATCH 09/24] adding catkin_pip function to call rosdep install. attempting fix for pypackage tests. --- CMakeLists.txt | 1 + cmake/catkin-pip-prefix.cmake.in | 4 --- cmake/catkin-pip-rosdep-install.cmake.in | 34 ++++++++++++++++++++++++ cmake/catkin-pip.cmake.in | 6 +++++ test/pypackage/CMakeLists.txt | 3 +++ 5 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 cmake/catkin-pip-rosdep-install.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b07a86..df1a093 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,6 +35,7 @@ catkin_package( catkin-pip-package.cmake catkin-pip-prefix.cmake catkin-pip-requirements.cmake + catkin-pip-rosdep-install.cmake pytest.cmake nosetests.cmake ) diff --git a/cmake/catkin-pip-prefix.cmake.in b/cmake/catkin-pip-prefix.cmake.in index 6d90c2a..61fb398 100644 --- a/cmake/catkin-pip-prefix.cmake.in +++ b/cmake/catkin-pip-prefix.cmake.in @@ -116,10 +116,6 @@ endfunction(find_catkin_system_pip) function(catkin_pip_setup_prefix ws_prefix) - # Setting up our environment (for devel space only) - #catkin_pip_check_env(${ws_prefix}) - # TODO : make this an env hook and use it via CATKIN_ENV - # Trying to find our own pip # Careful this creates a CACHE variable that we need to recreate here in case people clean devel without cleaning build unset(CATKIN_PIP CACHE) diff --git a/cmake/catkin-pip-rosdep-install.cmake.in b/cmake/catkin-pip-rosdep-install.cmake.in new file mode 100644 index 0000000..51725ac --- /dev/null +++ b/cmake/catkin-pip-rosdep-install.cmake.in @@ -0,0 +1,34 @@ +message(STATUS "Loading catkin-pip-rosdep-install.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") + +if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) + message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) +endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) + +# Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard +if(catkin_pip_rosdep_install_included) + return() +endif(catkin_pip_rosdep_install_included) +set(catkin_pip_rosdep_install_included true) + +# protecting against missing cmake file +include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-prefix.cmake" RESULT_VARIABLE CATKIN_PIP_PREFIX_FOUND ) +IF ( NOT CATKIN_PIP_PREFIX_FOUND ) + message ( FATAL_ERROR "{CMAKE_CURRENT_LIST_DIR}/catkin-pip-prefix.cmake Not Found !!!" ) +ENDIF ( NOT CATKIN_PIP_PREFIX_FOUND ) + +# +# Here we also setup the environment so that detected pip version is usable enough +# This is used by any other project directly (at configure time). +# It will automatically setup a prefix with catkin-pip. +# +function(catkin_pip_rosdep_install) + + # Setting up our environment (for devel space only) + # Needed in case user call this directly (configure time) + catkin_pip_setup_prefix(${CATKIN_PIP_ENV}) + + # runnig the pip command (configure time) + #catkin_pip_runcmd(${CATKIN_PIP} install ${ARGN} -r ${requirements_txt} --ignore-installed --src ${CMAKE_SOURCE_DIR} --exists-action b --prefix "${CATKIN_DEVEL_PREFIX}") + catkin_pip_runcmd(rosdep install --from-paths "${CMAKE_CURRENT_SOURCE_DIR}" -y) + +endfunction() diff --git a/cmake/catkin-pip.cmake.in b/cmake/catkin-pip.cmake.in index 3940bbc..7a87860 100644 --- a/cmake/catkin-pip.cmake.in +++ b/cmake/catkin-pip.cmake.in @@ -18,6 +18,12 @@ if(catkin_pip_main_included) endif(catkin_pip_main_included) set(catkin_pip_main_included true) +# protecting against missing cmake file dependency +include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-rosdep-install.cmake" RESULT_VARIABLE CATKIN_PIP_ROSDEP_INSTALL_FOUND ) +IF ( NOT CATKIN_PIP_ROSDEP_INSTALL_FOUND ) + message ( FATAL_ERROR "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-rosdep-install.cmake Not Found !!!" ) +ENDIF ( NOT CATKIN_PIP_ROSDEP_INSTALL_FOUND ) + # protecting against missing cmake file dependency include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-requirements.cmake" RESULT_VARIABLE CATKIN_PIP_REQUIREMENTS_FOUND ) IF ( NOT CATKIN_PIP_REQUIREMENTS_FOUND ) diff --git a/test/pypackage/CMakeLists.txt b/test/pypackage/CMakeLists.txt index 9a08c05..1e69b17 100644 --- a/test/pypackage/CMakeLists.txt +++ b/test/pypackage/CMakeLists.txt @@ -9,6 +9,9 @@ set (PIP_PROJECT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python_boilerplate) find_package(catkin REQUIRED COMPONENTS catkin_pip) +# Installing ROS dependencies from package.xml +catkin_pip_rosdep_install() + # For development, we need to install the pip dependencies in the workspace being created # Upon install, this will not do anything. All requirements should be satisfied by ROS dependencies system. # => pip dependencies will work from source only From ee77f5f144b70946715ebb3741c80dabc0f92231 Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 31 Jan 2017 16:33:49 +0900 Subject: [PATCH 10/24] adding libssl-dev as dependency forpypackage --- test/pypackage/package.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/pypackage/package.xml b/test/pypackage/package.xml index e88b1af..bac2af2 100644 --- a/test/pypackage/package.xml +++ b/test/pypackage/package.xml @@ -12,8 +12,9 @@ catkin catkin_pip - + libffi-dev + libssl-dev From 6c0d7599c28e3497cb649ccccd8bff7b5e125159 Mon Sep 17 00:00:00 2001 From: AlexV Date: Wed, 1 Feb 2017 20:07:30 +0900 Subject: [PATCH 11/24] improved concurrency handling for pip by using setlock from daemontools. improved status message output. --- cmake/catkin-pip-package.cmake.in | 5 ++- cmake/catkin-pip-prefix.cmake.in | 16 ++++---- cmake/catkin-pip-requirements.cmake.in | 5 ++- cmake/catkin-pip-rosdep-install.cmake.in | 5 ++- cmake/catkin-pip-runcmd.cmake.in | 41 ++++++++----------- cmake/catkin-pip.cmake.in | 16 ++++---- .../42.site_packages.sh.develspace.in | 1 + cmake/pytest.cmake.in | 5 --- package.xml | 5 +++ test/catkin_pip_pytest/test_easyinstallpth.py | 3 +- test/catkin_pip_pytest/test_site.py | 2 + 11 files changed, 54 insertions(+), 50 deletions(-) diff --git a/cmake/catkin-pip-package.cmake.in b/cmake/catkin-pip-package.cmake.in index d027e4c..bee9d6b 100644 --- a/cmake/catkin-pip-package.cmake.in +++ b/cmake/catkin-pip-package.cmake.in @@ -1,5 +1,3 @@ -message(STATUS "Loading catkin-pip-package.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") - if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) @@ -10,6 +8,9 @@ if(catkin_pip_setup_included) endif(catkin_pip_setup_included) set(catkin_pip_setup_included true) +message(STATUS "Loading catkin-pip-package.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") + + # protecting against missing cmake file include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-runcmd.cmake" RESULT_VARIABLE CATKIN_PIP_RUNCMD_FOUND ) IF ( NOT CATKIN_PIP_RUNCMD_FOUND ) diff --git a/cmake/catkin-pip-prefix.cmake.in b/cmake/catkin-pip-prefix.cmake.in index 61fb398..bfcc3cf 100644 --- a/cmake/catkin-pip-prefix.cmake.in +++ b/cmake/catkin-pip-prefix.cmake.in @@ -1,20 +1,22 @@ -message(STATUS "Loading catkin-pip-prefix.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") - if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) -# -# Important : This script is included by multiple cmake scripts at configure and build time -# So it needs to be idempotent (no change if called multiple time with same settings/environment) -# - # Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard if(catkin_pip_prefix_included) return() endif(catkin_pip_prefix_included) set(catkin_pip_prefix_included true) +message(STATUS "Loading catkin-pip-prefix.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") + + +# +# Important : This script is included by multiple cmake scripts at configure and build time +# So it needs to be idempotent (no change if called multiple time with same settings/environment) +# + + # protecting against missing cmake file include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-runcmd.cmake" RESULT_VARIABLE CATKIN_PIP_RUNCMD_FOUND ) IF ( NOT CATKIN_PIP_RUNCMD_FOUND ) diff --git a/cmake/catkin-pip-requirements.cmake.in b/cmake/catkin-pip-requirements.cmake.in index 38aef06..b2baf03 100644 --- a/cmake/catkin-pip-requirements.cmake.in +++ b/cmake/catkin-pip-requirements.cmake.in @@ -1,5 +1,3 @@ -message(STATUS "Loading catkin-pip-requirements.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") - if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) @@ -10,6 +8,9 @@ if(catkin_pip_requirements_included) endif(catkin_pip_requirements_included) set(catkin_pip_requirements_included true) +message(STATUS "Loading catkin-pip-requirements.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") + + # protecting against missing cmake file include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-prefix.cmake" RESULT_VARIABLE CATKIN_PIP_PREFIX_FOUND ) IF ( NOT CATKIN_PIP_PREFIX_FOUND ) diff --git a/cmake/catkin-pip-rosdep-install.cmake.in b/cmake/catkin-pip-rosdep-install.cmake.in index 51725ac..9004c35 100644 --- a/cmake/catkin-pip-rosdep-install.cmake.in +++ b/cmake/catkin-pip-rosdep-install.cmake.in @@ -1,5 +1,3 @@ -message(STATUS "Loading catkin-pip-rosdep-install.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") - if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) @@ -10,6 +8,9 @@ if(catkin_pip_rosdep_install_included) endif(catkin_pip_rosdep_install_included) set(catkin_pip_rosdep_install_included true) +message(STATUS "Loading catkin-pip-rosdep-install.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") + + # protecting against missing cmake file include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-prefix.cmake" RESULT_VARIABLE CATKIN_PIP_PREFIX_FOUND ) IF ( NOT CATKIN_PIP_PREFIX_FOUND ) diff --git a/cmake/catkin-pip-runcmd.cmake.in b/cmake/catkin-pip-runcmd.cmake.in index 0181314..5d7ae6c 100644 --- a/cmake/catkin-pip-runcmd.cmake.in +++ b/cmake/catkin-pip-runcmd.cmake.in @@ -1,20 +1,20 @@ -message(STATUS "Loading catkin-pip-runcmd.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") - if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) -# -# Important : This script is included by multiple cmake scripts at configure and build time -# So it needs to be idempotent (no change if called multiple time with same settings/environment) -# - # Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard if(catkin_pip_runcmd_included) return() endif(catkin_pip_runcmd_included) set(catkin_pip_runcmd_included true) +message(STATUS "Loading catkin-pip-runcmd.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") + +# +# Important : This script is included by multiple cmake scripts at configure and build time +# So it needs to be idempotent (no change if called multiple time with same settings/environment) +# + function(catkin_pip_runcmd) @@ -48,32 +48,27 @@ function(catkin_pip_install_devel_target package_name package_path) ${CATKIN_DEVEL_PREFIX}/${CATKIN_PIP_PYTHON_INSTALL_DIR}/${package_name}.egg-link ) - set(CATKIN_PIP_COMMAND ${CATKIN_PIP} install -e ${package_path} --prefix "${CATKIN_DEVEL_PREFIX}" ${ARGN}) - # TODO : change to use --target to allow installing duplicate packages... Ref https://github.com/pypa/pip/issues/4243 + # note setlock + set(CATKIN_PIP_COMMAND "${CATKIN_PIP}" install -e "${package_path}" --prefix "${CATKIN_DEVEL_PREFIX}" ${ARGN}) + # TODO : change to use --target to allow installing duplicate packages with pip>=9.0.0 ... Ref https://github.com/pypa/pip/issues/4243 - string(REPLACE ";" " " CATKIN_PIP_CMDSTR "${CATKIN_ENV} ${CATKIN_PIP_COMMAND}") + string(REPLACE ";" " " CATKIN_PIP_CMDSTR "${CATKIN_ENV} setlock ${CATKIN_PIP_ENV}/catkin_pip.lock ${CATKIN_PIP_COMMAND}") + + # Note that catkin_env is called first. That is because we do not trust catkin_env return code and async behavior. + # However we have to assume it doesn't alter the environment in a way that breaks setlock. + # Hint : on ubuntu trusty, I noticed the lock file stay there after execution, however the mutex behavior works fine. add_custom_command( OUTPUT ${CATKIN_PIP_INSTALL_DEVEL_OUTPUTS} + # Note we are using setlock to avoid concurrent pip calls (if make runs target in parallel) # Note we need to use catkin_env to make sure our envhook is properly loaded - COMMAND ${CATKIN_ENV} ${CATKIN_PIP_COMMAND} + COMMAND ${CATKIN_ENV} setlock "${CATKIN_PIP_ENV}/catkin_pip.lock" ${CATKIN_PIP_COMMAND} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS "${package_path}" # file level dependency : package needs to be in the path COMMENT "${CATKIN_PIP_CMDSTR}" VERBATIM ) - add_custom_target(${package_name} DEPENDS ${CATKIN_PIP_INSTALL_DEVEL_OUTPUTS}) # file level dependency : we need the output of the custom command - - if (CATKIN_PIP_DEVEL_TARGETS) - add_dependencies(${package_name} ${CATKIN_PIP_DEVEL_TARGETS}) # target level dependency to prevent concurrency - endif() - - # To have all target install one after the other (even if user attempts concurrent run) - # since pip doesnt support concurrent runs - set( CATKIN_PIP_DEVEL_TARGETS - ${CATKIN_PIP_DEVEL_TARGETS} ${package_name} - CACHE INTERNAL "List of targets for pip for devel workspace" - ) + add_custom_target(${package_name} ALL DEPENDS ${CATKIN_PIP_INSTALL_DEVEL_OUTPUTS}) # file level dependency : we need the output of the custom command endfunction() diff --git a/cmake/catkin-pip.cmake.in b/cmake/catkin-pip.cmake.in index 7a87860..845e6fb 100644 --- a/cmake/catkin-pip.cmake.in +++ b/cmake/catkin-pip.cmake.in @@ -1,9 +1,15 @@ -message(STATUS "Loading catkin-pip.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") - if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) +# Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard +if(catkin_pip_main_included) + return() +endif(catkin_pip_main_included) +set(catkin_pip_main_included true) + +message(STATUS "Loading catkin-pip.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") + # Setting required policies # required to be able to do "if (True)" # FOREACH(policy CMP0011 CMP0012 CMP0013 CMP0014) @@ -12,12 +18,6 @@ endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) # ENDIF() # ENDFOREACH() -# Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard -if(catkin_pip_main_included) - return() -endif(catkin_pip_main_included) -set(catkin_pip_main_included true) - # protecting against missing cmake file dependency include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-rosdep-install.cmake" RESULT_VARIABLE CATKIN_PIP_ROSDEP_INSTALL_FOUND ) IF ( NOT CATKIN_PIP_ROSDEP_INSTALL_FOUND ) diff --git a/cmake/env-hooks/42.site_packages.sh.develspace.in b/cmake/env-hooks/42.site_packages.sh.develspace.in index 94e3b15..1410f1e 100755 --- a/cmake/env-hooks/42.site_packages.sh.develspace.in +++ b/cmake/env-hooks/42.site_packages.sh.develspace.in @@ -36,6 +36,7 @@ PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/ # Prepending here our easy-install.pth content into PYTHONPATH, for overlayed workspace to get it before the workspace path # Somehow Python pth/.egg behavior doesnt work well with workspace overlays, based on PYTHONPATH. # However be aware of : https://github.com/pypa/pip/issues/3160 +# And : https://github.com/pypa/pip/issues/4261 if [ -f "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" ];then while read p; do #echo "Inserting $p before "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" into PYTHONPATH" diff --git a/cmake/pytest.cmake.in b/cmake/pytest.cmake.in index 6c5dd34..32cf7c2 100644 --- a/cmake/pytest.cmake.in +++ b/cmake/pytest.cmake.in @@ -1,10 +1,5 @@ _generate_function_if_testing_is_disabled("catkin_add_pytests") -include ( "${CMAKE_CURRENT_LIST_DIR}/nosetests.cmake" RESULT_VARIABLE CATKIN_PIP_NOSETESTS_FOUND ) -IF ( NOT CATKIN_PIP_NOSETESTS_FOUND ) - message ( FATAL_ERROR "${CMAKE_CURRENT_LIST_DIR}/nosetests.cmake Not Found !!!" ) -ENDIF ( NOT CATKIN_PIP_NOSETESTS_FOUND ) - # # Add Python pytest tests. # diff --git a/package.xml b/package.xml index 714ddbc..8b02fca 100644 --- a/package.xml +++ b/package.xml @@ -17,6 +17,7 @@ catkin python + @@ -25,5 +26,9 @@ python-pip + + + daemontools + git diff --git a/test/catkin_pip_pytest/test_easyinstallpth.py b/test/catkin_pip_pytest/test_easyinstallpth.py index cbecdd5..1082466 100644 --- a/test/catkin_pip_pytest/test_easyinstallpth.py +++ b/test/catkin_pip_pytest/test_easyinstallpth.py @@ -18,4 +18,5 @@ def test_easyinstall_content(devel_space, git_working_tree): ]: assert os.path.join(git_working_tree, 'test', python_pkg) in editable_paths, "{0} not in easy-install.pth".format(os.path.join(git_working_tree, 'test', python_pkg)) - + # Note if not in the file, it means it s already in PYTHONPATH : Ref https://github.com/pypa/pip/issues/4261 + # Get pip fix, or fix this test for it... diff --git a/test/catkin_pip_pytest/test_site.py b/test/catkin_pip_pytest/test_site.py index 84cff57..dc64cdf 100644 --- a/test/catkin_pip_pytest/test_site.py +++ b/test/catkin_pip_pytest/test_site.py @@ -11,3 +11,5 @@ def test_site(): for p in site.getsitepackages(): assert p in sys.path, "site packages directory {p} not found in {sys.path}".format(**locals()) + +# TODO : verify that "python -m site" from the shell actually would return something, after sourcing setup.sh => HOW ? From b92c4b1cc69883c843a3a71de536fefd10f819d3 Mon Sep 17 00:00:00 2001 From: AlexV Date: Thu, 2 Feb 2017 09:49:20 +0900 Subject: [PATCH 12/24] moved rosdep install out of catkin_pip, only used for tests. temporarily skipping broken test because of pip behavior... --- CMakeLists.txt | 1 - cmake/catkin-pip-rosdep-install.cmake.in | 35 ------------------- cmake/catkin-pip.cmake.in | 6 ---- test/CMakeLists.txt | 6 ++++ test/catkin_pip_pytest/test_easyinstallpth.py | 4 ++- test/pypackage/CMakeLists.txt | 3 -- 6 files changed, 9 insertions(+), 46 deletions(-) delete mode 100644 cmake/catkin-pip-rosdep-install.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index df1a093..7b07a86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,6 @@ catkin_package( catkin-pip-package.cmake catkin-pip-prefix.cmake catkin-pip-requirements.cmake - catkin-pip-rosdep-install.cmake pytest.cmake nosetests.cmake ) diff --git a/cmake/catkin-pip-rosdep-install.cmake.in b/cmake/catkin-pip-rosdep-install.cmake.in deleted file mode 100644 index 9004c35..0000000 --- a/cmake/catkin-pip-rosdep-install.cmake.in +++ /dev/null @@ -1,35 +0,0 @@ -if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) - message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" ) -endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 ) - -# Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard -if(catkin_pip_rosdep_install_included) - return() -endif(catkin_pip_rosdep_install_included) -set(catkin_pip_rosdep_install_included true) - -message(STATUS "Loading catkin-pip-rosdep-install.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") - - -# protecting against missing cmake file -include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-prefix.cmake" RESULT_VARIABLE CATKIN_PIP_PREFIX_FOUND ) -IF ( NOT CATKIN_PIP_PREFIX_FOUND ) - message ( FATAL_ERROR "{CMAKE_CURRENT_LIST_DIR}/catkin-pip-prefix.cmake Not Found !!!" ) -ENDIF ( NOT CATKIN_PIP_PREFIX_FOUND ) - -# -# Here we also setup the environment so that detected pip version is usable enough -# This is used by any other project directly (at configure time). -# It will automatically setup a prefix with catkin-pip. -# -function(catkin_pip_rosdep_install) - - # Setting up our environment (for devel space only) - # Needed in case user call this directly (configure time) - catkin_pip_setup_prefix(${CATKIN_PIP_ENV}) - - # runnig the pip command (configure time) - #catkin_pip_runcmd(${CATKIN_PIP} install ${ARGN} -r ${requirements_txt} --ignore-installed --src ${CMAKE_SOURCE_DIR} --exists-action b --prefix "${CATKIN_DEVEL_PREFIX}") - catkin_pip_runcmd(rosdep install --from-paths "${CMAKE_CURRENT_SOURCE_DIR}" -y) - -endfunction() diff --git a/cmake/catkin-pip.cmake.in b/cmake/catkin-pip.cmake.in index 845e6fb..ca1ad8f 100644 --- a/cmake/catkin-pip.cmake.in +++ b/cmake/catkin-pip.cmake.in @@ -18,12 +18,6 @@ message(STATUS "Loading catkin-pip.cmake from ${CMAKE_CURRENT_LIST_DIR}... ") # ENDIF() # ENDFOREACH() -# protecting against missing cmake file dependency -include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-rosdep-install.cmake" RESULT_VARIABLE CATKIN_PIP_ROSDEP_INSTALL_FOUND ) -IF ( NOT CATKIN_PIP_ROSDEP_INSTALL_FOUND ) - message ( FATAL_ERROR "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-rosdep-install.cmake Not Found !!!" ) -ENDIF ( NOT CATKIN_PIP_ROSDEP_INSTALL_FOUND ) - # protecting against missing cmake file dependency include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-requirements.cmake" RESULT_VARIABLE CATKIN_PIP_REQUIREMENTS_FOUND ) IF ( NOT CATKIN_PIP_REQUIREMENTS_FOUND ) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9421815..134ef17 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -81,16 +81,22 @@ endif() # and make sure they still work with catkin by building them all as subprojects # + # Testing project itself +# We need to call rosdep for each packages (to install dependencies if there are any) +catkin_pip_runcmd(rosdep install --from-paths "pipproject" -y) add_subdirectory(pipproject) # Testing project itself +catkin_pip_runcmd(rosdep install --from-paths "pylibrary" -y) add_subdirectory(pylibrary) # Testing project itself +catkin_pip_runcmd(rosdep install --from-paths "pypackage" -y) add_subdirectory(pypackage) # Testing project itself +catkin_pip_runcmd(rosdep install --from-paths "pypackage-minimal" -y) add_subdirectory(pypackage-minimal) diff --git a/test/catkin_pip_pytest/test_easyinstallpth.py b/test/catkin_pip_pytest/test_easyinstallpth.py index 1082466..5cf7a1b 100644 --- a/test/catkin_pip_pytest/test_easyinstallpth.py +++ b/test/catkin_pip_pytest/test_easyinstallpth.py @@ -2,8 +2,10 @@ from __future__ import print_function import os +import pytest +@pytest.mark.skip(reason="Test broken because of https://github.com/pypa/pip/issues/4261") def test_easyinstall_content(devel_space, git_working_tree): easy_install_pth_path = os.path.join(devel_space, 'lib','python2.7','site-packages','easy-install.pth') assert os.path.exists(easy_install_pth_path) @@ -19,4 +21,4 @@ def test_easyinstall_content(devel_space, git_working_tree): assert os.path.join(git_working_tree, 'test', python_pkg) in editable_paths, "{0} not in easy-install.pth".format(os.path.join(git_working_tree, 'test', python_pkg)) # Note if not in the file, it means it s already in PYTHONPATH : Ref https://github.com/pypa/pip/issues/4261 - # Get pip fix, or fix this test for it... + # Fix this test for it, at least until pip fixes goes in... diff --git a/test/pypackage/CMakeLists.txt b/test/pypackage/CMakeLists.txt index 1e69b17..9a08c05 100644 --- a/test/pypackage/CMakeLists.txt +++ b/test/pypackage/CMakeLists.txt @@ -9,9 +9,6 @@ set (PIP_PROJECT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python_boilerplate) find_package(catkin REQUIRED COMPONENTS catkin_pip) -# Installing ROS dependencies from package.xml -catkin_pip_rosdep_install() - # For development, we need to install the pip dependencies in the workspace being created # Upon install, this will not do anything. All requirements should be satisfied by ROS dependencies system. # => pip dependencies will work from source only From 25bb9d09d529bb0122dc5ba40625208cdc024cd1 Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 17 Feb 2017 16:06:33 +0900 Subject: [PATCH 13/24] removed useless bash envhook, fixed easy-install parse to include last line even if no empty line at end of file. --- CMakeLists.txt | 2 -- cmake/env-hooks/42.site_packages.bash.develspace.in | 12 ------------ cmake/env-hooks/42.site_packages.sh.develspace.in | 6 +++--- 3 files changed, 3 insertions(+), 17 deletions(-) delete mode 100755 cmake/env-hooks/42.site_packages.bash.develspace.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b07a86..9af2cac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,6 @@ configure_file(cmake/catkin-pip-fixups.req ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKA configure_file(cmake/scripts/path_prepend.sh ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/scripts/path_prepend.sh COPYONLY) configure_file(cmake/env-hooks/42.site_packages.sh.installspace.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks/42.site_packages.sh.installspace.in COPYONLY) configure_file(cmake/env-hooks/42.site_packages.sh.develspace.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks/42.site_packages.sh.develspace.in COPYONLY) -configure_file(cmake/env-hooks/42.site_packages.bash.develspace.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks/42.site_packages.bash.develspace.in COPYONLY) #configure_file(cmake/templates/python_setuptools_install.bat.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/templates/python_setuptools_install.bat.in COPYONLY) configure_file(cmake/templates/python_setuptools_install.sh.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/templates/python_setuptools_install.sh.in COPYONLY) @@ -67,7 +66,6 @@ install(PROGRAMS install(FILES cmake/env-hooks/42.site_packages.sh.develspace.in cmake/env-hooks/42.site_packages.sh.installspace.in - cmake/env-hooks/42.site_packages.bash.develspace.in DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks ) diff --git a/cmake/env-hooks/42.site_packages.bash.develspace.in b/cmake/env-hooks/42.site_packages.bash.develspace.in deleted file mode 100755 index 6bfe91b..0000000 --- a/cmake/env-hooks/42.site_packages.bash.develspace.in +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# Careful : env.sh *executes* it during cmake configure, but setup.sh source it ! -# If using a sh script, this can make a lot of issues... -# Reference to address these from a Bourne shell http://stackoverflow.com/a/29835459 - -# For catkin it is simpler to do add site-packages to PYTHONPATH in _setup_util.py.in than us doing it from here... -# but until catkin does this by default, the proper way from outside catkin is probably a env-hook - -# Using the sh script that should be sitting just besides this one -ENV_HOOKS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -. ${ENV_HOOKS_DIR}/42.site_packages.sh \ No newline at end of file diff --git a/cmake/env-hooks/42.site_packages.sh.develspace.in b/cmake/env-hooks/42.site_packages.sh.develspace.in index 1410f1e..f0174a8 100755 --- a/cmake/env-hooks/42.site_packages.sh.develspace.in +++ b/cmake/env-hooks/42.site_packages.sh.develspace.in @@ -38,9 +38,9 @@ PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/ # However be aware of : https://github.com/pypa/pip/issues/3160 # And : https://github.com/pypa/pip/issues/4261 if [ -f "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" ];then - while read p; do - #echo "Inserting $p before "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" into PYTHONPATH" - PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "$p" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") + while IFS= read -r DEVEL_PKG_PATH || [[ -n "DEVEL_PKG_PATH" ]]; do + #echo "Inserting $DEVEL_PKG_PATH before "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" into PYTHONPATH" + PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "$DEVEL_PKG_PATH" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") done <"@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" fi From 448d71f12602a84922245650a6e2262c2b74e01d Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 17 Feb 2017 17:21:52 +0900 Subject: [PATCH 14/24] reviewing shell script to make them a tiny more robust... --- cmake/catkin-pip.cmake.in | 5 +-- .../42.site_packages.sh.develspace.in | 33 ++++++++++++++----- cmake/scripts/path_prepend.sh | 14 ++++++-- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/cmake/catkin-pip.cmake.in b/cmake/catkin-pip.cmake.in index ca1ad8f..d5748d9 100644 --- a/cmake/catkin-pip.cmake.in +++ b/cmake/catkin-pip.cmake.in @@ -56,10 +56,7 @@ file(MAKE_DIRECTORY ${CATKIN_DEVEL_PREFIX}/@CATKIN_PIP_GLOBAL_PYTHON_DESTINATION # Since we need the envhook as soon as a package is using catkin-pip from source. catkin_add_env_hooks(42.site_packages SHELLS sh DIRECTORY ${CATKIN_PIP_ENV_HOOKS_PATH}) -# TODO : Also add envhooks for other shells -catkin_add_env_hooks(42.site_packages SHELLS bash DIRECTORY ${CATKIN_PIP_ENV_HOOKS_PATH} SKIP_INSTALL) -# SKIP_INSTALL because we definitely do not want it ending up in all deb packages built with catkin_pip. -# If we need paths in install space, they should somehow be stored in generated scripts (make or others) +# If we need paths in install space, they should somehow be stored in generated scripts (make or others) as much as possible # TODO : cmake should include this file only once per workspace, so the env hook is not recreated for every package that uses catkin_pip if they are in same workspace... diff --git a/cmake/env-hooks/42.site_packages.sh.develspace.in b/cmake/env-hooks/42.site_packages.sh.develspace.in index f0174a8..24eff24 100755 --- a/cmake/env-hooks/42.site_packages.sh.develspace.in +++ b/cmake/env-hooks/42.site_packages.sh.develspace.in @@ -3,48 +3,63 @@ # If using a sh script, this can make a lot of issues... # Reference to address these from a Bourne shell http://stackoverflow.com/a/29835459 +# lets be safe : Ref : http://www.davidpashley.com/articles/writing-robust-shell-scripts/ +set -o nounset +set -o noglob +# set -o errexit # for development only +# tested with dash on ubuntu trusty + # For catkin it is simpler to do add site-packages to PYTHONPATH in _setup_util.py.in than us doing it from here... # but until catkin does this by default, the proper way from outside catkin is probably a env-hook like this one # Note : We should enable site-package only on devel space. # Install space should behave like packages, everything in debian layout. +# TODO : find a cleaner way to call a shell function defined in another script + # Setting up the catkin environment itself # This needs to be in a different prefix, to be able to generate an install dir with setuptools, yet clear of any catkin_pip artifacts. #echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") +PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails #echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ before @CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@ into PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}" "@CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@") +PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}" "@CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@") || ${PYTHONPATH} # to be safe if path_prepend fails # Adding bin/ into the path #echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@ in front of PATH" -PATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@" "${PATH}") +PATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@" "${PATH}") || ${PATH} # to be safe if path_prepend fails # Somehow It seems we need to also (re)add dist-packages here to get package tests to work with ROS packages in same workspace # Especially when running tests with catkin_pip from source... there seem to be some workflow issue due to catkin cache environment... # Anyway it doesnt hurt to have it here as well. # this is usual for catkin -> no echo. This might actually be a fix for catkin not setting this before running tests. #echo "Inserting @CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") +PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails # Problem : before which workspace ? we might break workspace overlay order here # We combine our build path (catkin_pip_env) with site-packages (pip default) location, before attempting to prepend to the python path # Careful : echo is only possible if we are in a bash script (not used by env.sh) #echo "Inserting @CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ before @CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@ into PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@") +PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@") || ${PYTHONPATH} # to be safe if path_prepend fails # Prepending here our easy-install.pth content into PYTHONPATH, for overlayed workspace to get it before the workspace path # Somehow Python pth/.egg behavior doesnt work well with workspace overlays, based on PYTHONPATH. # However be aware of : https://github.com/pypa/pip/issues/3160 # And : https://github.com/pypa/pip/issues/4261 -if [ -f "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" ];then - while IFS= read -r DEVEL_PKG_PATH || [[ -n "DEVEL_PKG_PATH" ]]; do +if [ -f "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" ]; then + # Ref : http://stackoverflow.com/questions/4165135/how-to-use-while-read-bash-to-read-the-last-line-in-a-file-if-there-s-no-new + while IFS='' read -r DEVEL_PKG_PATH || [ -n "$DEVEL_PKG_PATH" ]; do #echo "Inserting $DEVEL_PKG_PATH before "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" into PYTHONPATH" - PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "$DEVEL_PKG_PATH" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") + PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "$DEVEL_PKG_PATH" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") || ${PYTHONPATH} # to be safe if path_prepend fails done <"@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" fi # Exporting is required to make sure we get the new value in children processes (catkin_make, test runs, etc.) export PYTHONPATH -export PATH \ No newline at end of file +export PATH + + +# resetting shell options (especially useful if sourced) +set +o nounset +set +o noglob +set +o errexit # for development only \ No newline at end of file diff --git a/cmake/scripts/path_prepend.sh b/cmake/scripts/path_prepend.sh index 7b7ef52..6c37bd4 100755 --- a/cmake/scripts/path_prepend.sh +++ b/cmake/scripts/path_prepend.sh @@ -4,11 +4,21 @@ # and prepends it to the beginning of the second argument (a ':' separated list of path) # It returns the new list. +# lets be safe : Ref : http://www.davidpashley.com/articles/writing-robust-shell-scripts/ +set -o nounset +set -o noglob +# set -o errexit # for development only +# tested with dash on ubuntu trusty + if [ $# -gt 3 -o $# -lt 1 ]; then echo "Usage : path_prepend []" - exit 127 + exit 127 # TODO this breaks calling script (with source setup.bash) -> FIX IT fi if [ $# -ge 1 ]; then + if [ -z "$1" ]; then # to protect against empty arg + echo "ERROR: Attempting to prepend an empty argument" + exit 63 # TODO this breaks calling script (with source setup.bash) -> FIX IT (easy test by changing -z -> -n) + fi # The path to insert ARG="$1" fi @@ -34,7 +44,7 @@ else [ -z "${LIST:-}" ] && LIST=${ARG} || LIST=${LIST}:${ARG} #echo $LIST fi - if [ $p != $ARG ]; then + if [ X$p != X$ARG ]; then [ -z "${LIST:-}" ] && LIST=${p} || LIST=${LIST}:${p} #echo $LIST fi From 506574c97cbbd1fa6a4aac37b6f066aa11ae0fb4 Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 17 Feb 2017 19:42:35 +0900 Subject: [PATCH 15/24] commenting echoes from script since it cannot echo without breaking things anyway. --- cmake/scripts/path_prepend.sh | 4 ++-- test/catkin_pip_pytest/test_syspath.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmake/scripts/path_prepend.sh b/cmake/scripts/path_prepend.sh index 6c37bd4..9719b3f 100755 --- a/cmake/scripts/path_prepend.sh +++ b/cmake/scripts/path_prepend.sh @@ -11,12 +11,12 @@ set -o noglob # tested with dash on ubuntu trusty if [ $# -gt 3 -o $# -lt 1 ]; then - echo "Usage : path_prepend []" + # echo "Usage : path_prepend []" # TMP : we cannot echo anything in the way the script is used exit 127 # TODO this breaks calling script (with source setup.bash) -> FIX IT fi if [ $# -ge 1 ]; then if [ -z "$1" ]; then # to protect against empty arg - echo "ERROR: Attempting to prepend an empty argument" + # echo "ERROR: Attempting to prepend an empty argument" # TMP : we cannot echo anything in the way the script is used exit 63 # TODO this breaks calling script (with source setup.bash) -> FIX IT (easy test by changing -z -> -n) fi # The path to insert diff --git a/test/catkin_pip_pytest/test_syspath.py b/test/catkin_pip_pytest/test_syspath.py index 88e17e4..cd70055 100644 --- a/test/catkin_pip_pytest/test_syspath.py +++ b/test/catkin_pip_pytest/test_syspath.py @@ -43,7 +43,8 @@ def test_sys_path_editable(git_working_tree, devel_space): assert ss in sys.path, "{p} not in {sp}".format(p=ss, sp=sys.path) # Lets keep it simple and rely on python way of handling easy-install.pth and egg-link before we modify it... - # It seems that by default the eggs path are aded after the pythonpaths... + # By default the eggs path are added after the pythonpaths. We need opposite behavior for ROS python (due to how devel workspace works) + # this is handle by both catkin_pip for ROS usage and pyros_setup for python usage. # if os.path.exists(os.path.join(devel_space, 'lib/python2.7/site-packages')): # assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/site-packages')) From a7ce93ef278dbc5e06f16aa1383dff20e70ad692 Mon Sep 17 00:00:00 2001 From: AlexV Date: Fri, 17 Feb 2017 19:52:09 +0900 Subject: [PATCH 16/24] more exhaustive test for sys_path --- test/catkin_pip_pytest/test_syspath.py | 41 ++++++++++++++++---------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/test/catkin_pip_pytest/test_syspath.py b/test/catkin_pip_pytest/test_syspath.py index cd70055..a6eee81 100644 --- a/test/catkin_pip_pytest/test_syspath.py +++ b/test/catkin_pip_pytest/test_syspath.py @@ -3,7 +3,6 @@ import os import sys -import site def test_sys_path(devel_space): @@ -37,23 +36,33 @@ def test_sys_path_editable(git_working_tree, devel_space): print(sys.path) # Verifying the pip editable installed package location is in python path - ss = os.path.join(git_working_tree, 'test', 'pipproject', 'mypippkg') - if os.path.exists(ss): - assert ss in sys.path, "{p} not in {sp}".format(p=ss, sp=sys.path) + for python_pkg in [ + os.path.join('pipproject', 'mypippkg'), + os.path.join('pylibrary', 'python-nameless', 'src'), + os.path.join('pypackage', 'python_boilerplate'), + os.path.join('pypackage-minimal', 'cookiecutter_pypackage_minimal'), + ]: - # Lets keep it simple and rely on python way of handling easy-install.pth and egg-link before we modify it... - # By default the eggs path are added after the pythonpaths. We need opposite behavior for ROS python (due to how devel workspace works) - # this is handle by both catkin_pip for ROS usage and pyros_setup for python usage. + ss = os.path.join(git_working_tree, 'test', python_pkg) - # if os.path.exists(os.path.join(devel_space, 'lib/python2.7/site-packages')): - # assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/site-packages')) - # - # if os.path.exists(os.path.join(devel_space, 'lib/python2.7/dist-packages')): - # assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/dist-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/dist-packages')) - # - # if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - # assert sys.path.index(ss) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2}".format(p1=ss, p2='/opt/ros/indigo/lib/python2.7/dist-packages') + if os.path.exists(ss): + assert ss in sys.path, "{p} not in {sp}".format(p=ss, sp=sys.path) + # By default the egg-links path are added after the pythonpaths. + # We need opposite behavior for ROS python (due to how devel workspace works) + # this is handle by both catkin_pip for ROS usage and pyros_setup for python usage. - #TODO : verify the position of the catkin_pip_env paths (bin? and site-packages) + if os.path.exists(os.path.join(devel_space, 'lib/python2.7/site-packages')): + assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/site-packages')) + + if os.path.exists(os.path.join(devel_space, 'lib/python2.7/dist-packages')): + assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/dist-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/dist-packages')) + + if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): + assert sys.path.index(ss) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2}".format(p1=ss, p2='/opt/ros/indigo/lib/python2.7/dist-packages') + + + #TODO : verify the position of the catkin_pip_env paths (bin? and site-packages) + + # TODO : check overlay / underlay order \ No newline at end of file From 6a680ea1a8da7ec8c3b311535e4ec92c39ebb826 Mon Sep 17 00:00:00 2001 From: AlexV Date: Mon, 20 Feb 2017 12:50:08 +0900 Subject: [PATCH 17/24] replacing CI_ROS_DISTRO var by ROS_DISTRO, to have it set for calls to rosdep. --- .travis.yml | 22 +++++++++++----------- travis_checks.bash | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index cfbc716..d04e00c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,19 +13,19 @@ env: # This will check any ROS distro supported on this OS # checking devel and install separately so that they don't influence each other (dependencies, path, env, etc.) matrix: - - CI_ROS_DISTRO=indigo ROS_FLOW=devel - - CI_ROS_DISTRO=indigo ROS_FLOW=install - - CI_ROS_DISTRO=jade ROS_FLOW=devel - - CI_ROS_DISTRO=jade ROS_FLOW=install - - CI_ROS_DISTRO=kinetic ROS_FLOW=devel - - CI_ROS_DISTRO=kinetic ROS_FLOW=install + - ROS_DISTRO=indigo ROS_FLOW=devel + - ROS_DISTRO=indigo ROS_FLOW=install + - ROS_DISTRO=jade ROS_FLOW=devel + - ROS_DISTRO=jade ROS_FLOW=install + - ROS_DISTRO=kinetic ROS_FLOW=devel + - ROS_DISTRO=kinetic ROS_FLOW=install # TODO : test all possible flows (installing deps on devel or not, etc.) before_install: # Getting docker ros image - - docker pull ros:${CI_ROS_DISTRO}-ros-core + - docker pull ros:${ROS_DISTRO}-ros-core # Running as daemon - - docker run --name ${CONTAINER_NAME} -d -t ros:${CI_ROS_DISTRO}-ros-core /bin/bash | tee container.id + - docker run --name ${CONTAINER_NAME} -d -t ros:${ROS_DISTRO}-ros-core /bin/bash | tee container.id # Checking current container - docker ps -a - docker exec -ti ${CONTAINER_NAME} hostname @@ -41,15 +41,15 @@ install: # copying local clone to the running container (volume is currently broken) - docker cp . ${CONTAINER_NAME}:/git_clone # Installing package dependencies - - docker exec -ti ${CONTAINER_NAME} rosdep install --default-yes --from-paths /git_clone --rosdistro $CI_ROS_DISTRO + - docker exec -ti ${CONTAINER_NAME} rosdep install --default-yes --from-paths /git_clone --rosdistro $ROS_DISTRO # full ROS setup, build and test script: - CONTAINER_ID=$(cat container.id) - docker ps -a - - docker exec -ti ${CONTAINER_NAME} /bin/bash -c "source /opt/ros/$CI_ROS_DISTRO/setup.bash && rospack profile" + - docker exec -ti ${CONTAINER_NAME} /bin/bash -c "source /opt/ros/$ROS_DISTRO/setup.bash && rospack profile" # Passing env vars here since passing in docker run currently breaks (2016-08-25) - - docker exec -ti ${CONTAINER_NAME} /bin/bash -c "export CI_ROS_DISTRO=$CI_ROS_DISTRO && export ROS_FLOW=$ROS_FLOW && /git_clone/travis_checks.bash" + - docker exec -ti ${CONTAINER_NAME} /bin/bash -c "export ROS_DISTRO=$ROS_DISTRO && export ROS_FLOW=$ROS_FLOW && /git_clone/travis_checks.bash" - docker stop "${CONTAINER_ID}" notifications: diff --git a/travis_checks.bash b/travis_checks.bash index 347ff91..62293a7 100755 --- a/travis_checks.bash +++ b/travis_checks.bash @@ -15,7 +15,7 @@ cd $DIR # For travis docker, this is already done by the entrypoint in docker image. # However when using 'docker exec' we still need to source it ourselves. # Also it is mandatory when this script is run directly by the developer. -source /opt/ros/$CI_ROS_DISTRO/setup.bash +source /opt/ros/$ROS_DISTRO/setup.bash mkdir -p testbuild cd testbuild From 9a08ae52373cc53f2f346184ce469da2e858729e Mon Sep 17 00:00:00 2001 From: AlexV Date: Mon, 20 Feb 2017 15:40:44 +0900 Subject: [PATCH 18/24] cleaning up command args when running pip install --editable to clean python path and workaround pip bug... --- cmake/catkin-pip-runcmd.cmake.in | 21 ++++++++++++++----- .../42.site_packages.sh.develspace.in | 2 +- .../42.site_packages.sh.installspace.in | 13 +++++++++++- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/cmake/catkin-pip-runcmd.cmake.in b/cmake/catkin-pip-runcmd.cmake.in index 5d7ae6c..fa19c3a 100644 --- a/cmake/catkin-pip-runcmd.cmake.in +++ b/cmake/catkin-pip-runcmd.cmake.in @@ -20,6 +20,7 @@ function(catkin_pip_runcmd) set(CATKIN_PIP_COMMAND ${ARGN}) + # TODO : refactor to use separate arguments, like catkin_pip_install_devel_target string(REPLACE ";" " " CATKIN_PIP_CMDSTR "${CATKIN_ENV} ${CATKIN_PIP_COMMAND}") message(STATUS " ... Running ${CATKIN_PIP_CMDSTR} ...") @@ -48,11 +49,18 @@ function(catkin_pip_install_devel_target package_name package_path) ${CATKIN_DEVEL_PREFIX}/${CATKIN_PIP_PYTHON_INSTALL_DIR}/${package_name}.egg-link ) - # note setlock - set(CATKIN_PIP_COMMAND "${CATKIN_PIP}" install -e "${package_path}" --prefix "${CATKIN_DEVEL_PREFIX}" ${ARGN}) + string(REPLACE ";" " " ARGN_STR "${ARGN}") + + # note setlock because of https://github.com/pypa/pip/issues/2361 + # set(CATKIN_PIP_CMD ${CATKIN_ENV}) + # set(CATKIN_PIP_CMD_ARGS_STR "setlock ${CATKIN_PIP_ENV}/catkin_pip.lock ${CATKIN_PIP} install -e ${package_path} --prefix ${CATKIN_DEVEL_PREFIX} ${ARGN_STR}") + # separate_arguments(CATKIN_PIP_CMD_ARGS_LIST UNIX_COMMAND ${CATKIN_PIP_CMD_ARGS_STR}) # TODO : change to use --target to allow installing duplicate packages with pip>=9.0.0 ... Ref https://github.com/pypa/pip/issues/4243 - string(REPLACE ";" " " CATKIN_PIP_CMDSTR "${CATKIN_ENV} setlock ${CATKIN_PIP_ENV}/catkin_pip.lock ${CATKIN_PIP_COMMAND}") + # TMP : since PYTHONPATH breaks pip install --editable Ref : https://github.com/pypa/pip/issues/4261 + set(CATKIN_PIP_CMD /bin/bash) # Ubuntu (Linux?) only + set(CATKIN_PIP_CMD_ARGS_STR "-c 'export PYTHONPATH=${CATKIN_DEVEL_PREFIX}/${CATKIN_PIP_PYTHON_INSTALL_DIR}\; setlock ${CATKIN_PIP_ENV}/catkin_pip.lock ${CATKIN_PIP} install -e ${package_path} --prefix ${CATKIN_DEVEL_PREFIX} ${ARGN_STR}'") + separate_arguments(CATKIN_PIP_CMD_ARGS_LIST UNIX_COMMAND ${CATKIN_PIP_CMD_ARGS_STR}) # Note that catkin_env is called first. That is because we do not trust catkin_env return code and async behavior. # However we have to assume it doesn't alter the environment in a way that breaks setlock. @@ -60,12 +68,15 @@ function(catkin_pip_install_devel_target package_name package_path) add_custom_command( OUTPUT ${CATKIN_PIP_INSTALL_DEVEL_OUTPUTS} + # Note we are using setlock to avoid concurrent pip calls (if make runs target in parallel) + # Note we override PYTHONPATH to workaround a pip bug # Note we need to use catkin_env to make sure our envhook is properly loaded - COMMAND ${CATKIN_ENV} setlock "${CATKIN_PIP_ENV}/catkin_pip.lock" ${CATKIN_PIP_COMMAND} + + COMMAND ${CATKIN_PIP_CMD} ${CATKIN_PIP_CMD_ARGS_LIST} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS "${package_path}" # file level dependency : package needs to be in the path - COMMENT "${CATKIN_PIP_CMDSTR}" + COMMENT "${CATKIN_PIP_CMD} ${CATKIN_PIP_CMD_ARGS_STR}" VERBATIM ) diff --git a/cmake/env-hooks/42.site_packages.sh.develspace.in b/cmake/env-hooks/42.site_packages.sh.develspace.in index 24eff24..6c9a65b 100755 --- a/cmake/env-hooks/42.site_packages.sh.develspace.in +++ b/cmake/env-hooks/42.site_packages.sh.develspace.in @@ -62,4 +62,4 @@ export PATH # resetting shell options (especially useful if sourced) set +o nounset set +o noglob -set +o errexit # for development only \ No newline at end of file +# set +o errexit # for development only \ No newline at end of file diff --git a/cmake/env-hooks/42.site_packages.sh.installspace.in b/cmake/env-hooks/42.site_packages.sh.installspace.in index 3cc240b..85badfe 100755 --- a/cmake/env-hooks/42.site_packages.sh.installspace.in +++ b/cmake/env-hooks/42.site_packages.sh.installspace.in @@ -3,6 +3,12 @@ # If using a sh script, this can make a lot of issues... # Reference to address these from a Bourne shell http://stackoverflow.com/a/29835459 +# lets be safe : Ref : http://www.davidpashley.com/articles/writing-robust-shell-scripts/ +set -o nounset +set -o noglob +# set -o errexit # for development only +# tested with dash on ubuntu trusty + # In install space we only need the catkin_pip environment setup. # Install space should behave like packages, everything in debian layout. @@ -18,4 +24,9 @@ PATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_GLO # Exporting is required to make sure we get the new value in children processes (catkin_make, test runs, etc.) export PYTHONPATH -export PATH \ No newline at end of file +export PATH + +# resetting shell options (especially useful if sourced) +set +o nounset +set +o noglob +# set +o errexit # for development only \ No newline at end of file From 4b25b2582d9a1eb8e5c8ed638580e9442db3268e Mon Sep 17 00:00:00 2001 From: AlexV Date: Mon, 20 Feb 2017 16:06:29 +0900 Subject: [PATCH 19/24] improved tests and fix all. added actual test for pytest to pipproject. --- test/CMakeLists.txt | 4 +- test/catkin_pip_pytest/conftest.py | 26 ++++++-- test/catkin_pip_pytest/test_import.py | 27 +++++++++ test/catkin_pip_pytest/test_path.py | 2 + test/catkin_pip_pytest/test_site.py | 4 +- test/catkin_pip_pytest/test_syspath.py | 83 +++++++++++++------------- test/pipproject/CMakeLists.txt | 3 +- test/pipproject/catkin_pip | 1 + 8 files changed, 101 insertions(+), 49 deletions(-) create mode 100644 test/catkin_pip_pytest/test_import.py create mode 100644 test/catkin_pip_pytest/test_path.py create mode 120000 test/pipproject/catkin_pip diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 134ef17..cad5324 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -34,7 +34,8 @@ endif() if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pipproject/mypippkg") message(STATUS " ... Getting cookiecutter-pipproject template from cookiecutter ...") execute_process( - COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/wdm0006/cookiecutter-pipproject.git + # TMP waiting for PR + COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/asmodehn/cookiecutter-pipproject.git --checkout adding_test_sample WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pipproject RESULT_VARIABLE PIP_RESULT OUTPUT_VARIABLE PIP_VARIABLE @@ -84,6 +85,7 @@ endif() # Testing project itself # We need to call rosdep for each packages (to install dependencies if there are any) +# note $ROS_DISTRO must be already defined in the environment catkin_pip_runcmd(rosdep install --from-paths "pipproject" -y) add_subdirectory(pipproject) diff --git a/test/catkin_pip_pytest/conftest.py b/test/catkin_pip_pytest/conftest.py index 11ad3ef..b8e978f 100644 --- a/test/catkin_pip_pytest/conftest.py +++ b/test/catkin_pip_pytest/conftest.py @@ -11,6 +11,21 @@ def build_dir(request): return request.config.getoption("--build-dir") +@pytest.fixture +def build_dir(request): + # current dir should be the build directory (binary dir for cmake) + # otherwise cmake tests commands are broken + return os.getcwd() + + +@pytest.fixture +def catkin_pip_env_dir(request): + # lets find catkin_pip_env space in current dir (cwd should be the build directory) + # expected setup.sh + cpe = os.path.join(build_dir(request), 'catkin_pip_env') + return cpe + + @pytest.fixture def git_working_tree(request): @@ -43,8 +58,11 @@ def devel_space(request): @pytest.fixture -def build_dir(request): - # current dir should be the build directory (binary dir for cmake) - # otherwise cmake tests commands are broken - return os.getcwd() +def install_space(request): + # lets find devel space in current dir (cwd should be the build directory) + # expected setup.sh + assert os.path.exists(os.path.abspath(os.path.join('install', 'setup.sh'))), "install/setup.bash not found in {0}".format(os.getcwd()) + ws = os.path.abspath('install') + return ws + diff --git a/test/catkin_pip_pytest/test_import.py b/test/catkin_pip_pytest/test_import.py new file mode 100644 index 0000000..8e0fc53 --- /dev/null +++ b/test/catkin_pip_pytest/test_import.py @@ -0,0 +1,27 @@ +from __future__ import absolute_import +from __future__ import print_function + +import os +import sys +import importlib + + +# If all package dependencies are in the sys.path, we can import them +def test_dynamic_import_devel_space_pkgs(): + importlib.import_module('click') # dependency from python_boilerplate + + +# If all packages are in the sys.path, we can import them +def test_dynamic_import_vedel_pkg(): + importlib.import_module('mypippkg') + importlib.import_module('nameless') + importlib.import_module('python_boilerplate') + importlib.import_module('cookiecutter_pypackage_minimal') + + +# If all catkin_pip dependencies are in the sys.path, we can import them +def test_dynamic_import_catkin_pip_env_pkgs(): + importlib.import_module('setuptools') + importlib.import_module('pip') + importlib.import_module('nose') + importlib.import_module('pytest') diff --git a/test/catkin_pip_pytest/test_path.py b/test/catkin_pip_pytest/test_path.py new file mode 100644 index 0000000..e7603b0 --- /dev/null +++ b/test/catkin_pip_pytest/test_path.py @@ -0,0 +1,2 @@ +# TODO : verify the position of the catkin_pip_env bin in path +# It has to be before the devel/bin path... \ No newline at end of file diff --git a/test/catkin_pip_pytest/test_site.py b/test/catkin_pip_pytest/test_site.py index dc64cdf..f6330ac 100644 --- a/test/catkin_pip_pytest/test_site.py +++ b/test/catkin_pip_pytest/test_site.py @@ -6,10 +6,10 @@ import site -# TODO : is this really useful ? -# we are not using any feature of site module itself (currently at least, and API is quite limited...) def test_site(): for p in site.getsitepackages(): assert p in sys.path, "site packages directory {p} not found in {sys.path}".format(**locals()) +# TODO : verify that site module behavior (HOW ? API limited,a nd sys.path is taken care of in another test file) + # TODO : verify that "python -m site" from the shell actually would return something, after sourcing setup.sh => HOW ? diff --git a/test/catkin_pip_pytest/test_syspath.py b/test/catkin_pip_pytest/test_syspath.py index a6eee81..5c43fc9 100644 --- a/test/catkin_pip_pytest/test_syspath.py +++ b/test/catkin_pip_pytest/test_syspath.py @@ -3,40 +3,59 @@ import os import sys +import pytest +import importlib -def test_sys_path(devel_space): +def in_sys_path_ordered(*arg_paths): + """ + asserts that path1 is before path2 in sys.path + :param arg_paths: the arguments are a list of path + :return: True if all paths appear in sys.path, and in the arguments order. False otherwise. + """ + return all(p in sys.path for p in arg_paths) and \ + all([sys.path.index(p) < sys.path.index(q) for p, q in zip(arg_paths,arg_paths[1:])] + if len(arg_paths) else True) + + +def test_devel_space_in_sys_path(devel_space): print(sys.path) # Verifying relative path order - if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - assert '/opt/ros/indigo/lib/python2.7/dist-packages' in sys.path, "{p} not in {sp}".format(p='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) - # TODO : It seems env_cached.sh append dist-packages path at the end of python_path somehow... We should investigate. - # HINT : only sourcing /opt/ros/indigo/setup.bash seems fine (dist-packages added at beginning of sys.path) - # HINT : env_cached.sh seems fine... maybe it comes from the way nosetests/py.test is launched ? - # HINT : following same process (building and running tests) with rospy_tutorials doesnt show the broken behavior. Something in catkin_pip breaking it ? - #assert len(sys.path) == len(set(sys.path)) # check unicity of paths + assert in_sys_path_ordered( + os.path.join(devel_space, 'lib/python2.7/site-packages'), + os.path.join(devel_space, 'lib/python2.7/dist-packages'), + ), "{site_pkgs} not appearing before {dist_pkgs} in sys.path".format( + site_pkgs=os.path.join(devel_space, 'lib/python2.7/site-packages'), + dist_pkgs=os.path.join(devel_space, 'lib/python2.7/dist-packages') + ) - if os.path.exists(os.path.join(devel_space, 'lib/python2.7/dist-packages')): - assert os.path.join(devel_space, 'lib/python2.7/dist-packages') in sys.path, "{p} not in {sp}".format(p=os.path.join(devel_space, 'lib/python2.7/dist-packages'), sp=sys.path) - if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - assert sys.path.index(os.path.join(devel_space, 'lib/python2.7/dist-packages')) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2} in {sp}".format(p1=os.path.join(devel_space, 'lib/python2.7/dist-packages'), p2='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) - if os.path.exists(os.path.join(devel_space, 'lib/python2.7/site-packages')): - assert os.path.join(devel_space, 'lib/python2.7/site-packages') in sys.path, "{p} not in {sp}".format(p=os.path.join(devel_space, 'lib/python2.7/site-packages'), sp=sys.path) - - if os.path.exists(os.path.join(devel_space, 'lib/python2.7/dist-packages')): - assert sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/dist-packages')), "{p1} not before {p2} in {sp}".format(p1=os.path.join(devel_space, 'lib/python2.7/site-packages'), p2=os.path.join(devel_space, 'lib/python2.7/dist-packages'), sp=sys.path) +def test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir): + print(sys.path) - if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - assert sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2} in {sp}".format(p1=os.path.join(devel_space, 'lib/python2.7/site-packages'), p2='/opt/ros/indigo/lib/python2.7/dist-packages', sp=sys.path) + # Verifying relative path order + assert in_sys_path_ordered( + os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages'), + ), "{site_pkgs} not appearing in sys.path".format( + site_pkgs=os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages') + ) def test_sys_path_editable(git_working_tree, devel_space): print(sys.path) - # Verifying the pip editable installed package location is in python path + def pkg_path_in_sys_path(pkg_path, before=[]): + assert os.path.exists(pkg_path), "{pkg_path} does not exist".format(**locals()) + + # By default the egg-links path are added after the pythonpaths. + # We need opposite behavior for ROS python (due to how devel workspace works) + # this is handle by both catkin_pip for ROS usage and pyros_setup for python usage. + + assert in_sys_path_ordered(*([pkg_path] + before)),\ + "paths not appearing in sys.path in the expected order : {p}".format(p=([pkg_path] + before)) + # Verifying the pip editable installed package location is in python path for python_pkg in [ os.path.join('pipproject', 'mypippkg'), os.path.join('pylibrary', 'python-nameless', 'src'), @@ -46,23 +65,7 @@ def test_sys_path_editable(git_working_tree, devel_space): ss = os.path.join(git_working_tree, 'test', python_pkg) - if os.path.exists(ss): - assert ss in sys.path, "{p} not in {sp}".format(p=ss, sp=sys.path) - - # By default the egg-links path are added after the pythonpaths. - # We need opposite behavior for ROS python (due to how devel workspace works) - # this is handle by both catkin_pip for ROS usage and pyros_setup for python usage. - - if os.path.exists(os.path.join(devel_space, 'lib/python2.7/site-packages')): - assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/site-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/site-packages')) - - if os.path.exists(os.path.join(devel_space, 'lib/python2.7/dist-packages')): - assert sys.path.index(ss) < sys.path.index(os.path.join(devel_space, 'lib/python2.7/dist-packages')), "{p1} not before {p2}".format(p1=ss, p2=os.path.join(devel_space, 'lib/python2.7/dist-packages')) - - if os.path.exists('/opt/ros/indigo/lib/python2.7/dist-packages'): - assert sys.path.index(ss) < sys.path.index('/opt/ros/indigo/lib/python2.7/dist-packages'), "{p1} not before {p2}".format(p1=ss, p2='/opt/ros/indigo/lib/python2.7/dist-packages') - - - #TODO : verify the position of the catkin_pip_env paths (bin? and site-packages) - - # TODO : check overlay / underlay order \ No newline at end of file + pkg_path_in_sys_path(ss, [ + os.path.join(devel_space, 'lib/python2.7/site-packages'), + os.path.join(devel_space, 'lib/python2.7/dist-packages') + ]) diff --git a/test/pipproject/CMakeLists.txt b/test/pipproject/CMakeLists.txt index a93cb97..862d198 100644 --- a/test/pipproject/CMakeLists.txt +++ b/test/pipproject/CMakeLists.txt @@ -25,6 +25,5 @@ catkin_pip_package(mypippkg ${PIP_PROJECT_DIR}) ## Unit tests if (CATKIN_ENABLE_TESTING) catkin_add_nosetests(${PIP_PROJECT_DIR}/tests DEPENDENCIES mypippkg) - # disabling pytest for now since it fails if there is no test detected (no tests implemented at the moment). - # catkin_add_pytests(${PIP_PROJECT_DIR}/tests DEPENDENCIES mypippkg) + catkin_add_pytests(${PIP_PROJECT_DIR}/tests DEPENDENCIES mypippkg) endif() diff --git a/test/pipproject/catkin_pip b/test/pipproject/catkin_pip new file mode 120000 index 0000000..0c9c92c --- /dev/null +++ b/test/pipproject/catkin_pip @@ -0,0 +1 @@ +../catkin_pip \ No newline at end of file From 05aa5a6ab888e629f707e84337099b11e4affb59 Mon Sep 17 00:00:00 2001 From: AlexV Date: Mon, 20 Feb 2017 18:46:54 +0900 Subject: [PATCH 20/24] fixed broken tests. now needs pyros-setup to get .pth paths before workspaces. cosmetics. --- cmake/catkin-pip-prefix.cmake.in | 3 +- .../42.site_packages.sh.develspace.in | 6 ++-- test/catkin_pip_pytest/test_easyinstallpth.py | 3 +- test/catkin_pip_pytest/test_syspath.py | 33 +++++++++++++++++-- test/test_requirements.txt | 3 +- travis_checks.bash | 4 +-- 6 files changed, 40 insertions(+), 12 deletions(-) diff --git a/cmake/catkin-pip-prefix.cmake.in b/cmake/catkin-pip-prefix.cmake.in index bfcc3cf..95d1f9d 100644 --- a/cmake/catkin-pip-prefix.cmake.in +++ b/cmake/catkin-pip-prefix.cmake.in @@ -124,12 +124,13 @@ function(catkin_pip_setup_prefix ws_prefix) find_program(CATKIN_PIP NAMES pip pip2 pip2.7 PATHS ${ws_prefix}/@CATKIN_GLOBAL_BIN_DESTINATION@ NO_DEFAULT_PATH) if (CATKIN_PIP) - message(STATUS " ... Catkin pip was found at ${CATKIN_PIP} ...") set(CATKIN_PIP "${CATKIN_PIP} -q") # we can add all default basic options here. else () # If not found, it means we need to do the whole setup... unset(CATKIN_PIP CACHE) + # TODO: cleanup if unnecessary message (the find_program already outputs something message(STATUS " ... Catkin pip was not found in ${ws_prefix}/@CATKIN_GLOBAL_BIN_DESTINATION@ ...") + # Assuming Ubuntu Trusty here. platform detection is another hurdle set(CMAKE_SYSTEM_PREFIX_PATH / /usr /usr/local) diff --git a/cmake/env-hooks/42.site_packages.sh.develspace.in b/cmake/env-hooks/42.site_packages.sh.develspace.in index 6c9a65b..46d0dbb 100755 --- a/cmake/env-hooks/42.site_packages.sh.develspace.in +++ b/cmake/env-hooks/42.site_packages.sh.develspace.in @@ -19,10 +19,8 @@ set -o noglob # Setting up the catkin environment itself # This needs to be in a different prefix, to be able to generate an install dir with setuptools, yet clear of any catkin_pip artifacts. -#echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails -#echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ before @CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@ into PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}" "@CATKIN_PIP_ENV@/@CATKIN_PYTHON_INSTALL_DIR@") || ${PYTHONPATH} # to be safe if path_prepend fails +#echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" +PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails # Adding bin/ into the path #echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@ in front of PATH" diff --git a/test/catkin_pip_pytest/test_easyinstallpth.py b/test/catkin_pip_pytest/test_easyinstallpth.py index 5cf7a1b..bd3fea1 100644 --- a/test/catkin_pip_pytest/test_easyinstallpth.py +++ b/test/catkin_pip_pytest/test_easyinstallpth.py @@ -5,9 +5,8 @@ import pytest -@pytest.mark.skip(reason="Test broken because of https://github.com/pypa/pip/issues/4261") def test_easyinstall_content(devel_space, git_working_tree): - easy_install_pth_path = os.path.join(devel_space, 'lib','python2.7','site-packages','easy-install.pth') + easy_install_pth_path = os.path.join(devel_space, 'lib', 'python2.7', 'site-packages', 'easy-install.pth') assert os.path.exists(easy_install_pth_path) with open(easy_install_pth_path) as easy_install_pth: editable_paths = easy_install_pth.read().splitlines() diff --git a/test/catkin_pip_pytest/test_syspath.py b/test/catkin_pip_pytest/test_syspath.py index 5c43fc9..48534c3 100644 --- a/test/catkin_pip_pytest/test_syspath.py +++ b/test/catkin_pip_pytest/test_syspath.py @@ -45,12 +45,41 @@ def test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir): def test_sys_path_editable(git_working_tree, devel_space): print(sys.path) + # IMPORTANT : Without pyros_setup configuration activated, + # the develop packages are set in sys.path AFTER the site-packages and dist-packages + # This is because of incompatibilities between ROS and python ways of handling PYTHONPATH. + # It is usually not a problem, unless : + # - you are working in a devel space with catkin_pip and expect to use a editable package from there + # - you have the same package installed somewhere else in the PYTHONPATH + # And in that case it might end up being *before* the source dir in sys.path + # + # => Without pyros_setup, the installed version will be preferred + # This is the python (and tools) way of working, since PYTHONPATH purpose is to override file-based setup, + # like editable packages. + # + # => With pyros_setup, the devel version will be preferred + # This is the ROS way of working, since PYTHONPATH order determine the order of folder when looking for a package. + # + # Both will work fine in most cases, but one might want to keep this corner case in mind... + + import pyros_setup + + # We need to pass the proper workspace here + pyros_setup.configurable_import().configure({ + 'WORKSPACES': [ + devel_space + ], # This should be the same as pytest devel_space fixture + 'DISTRO': pyros_setup.DETECTED_DISTRO, + }).activate() + + print(sys.path) + def pkg_path_in_sys_path(pkg_path, before=[]): assert os.path.exists(pkg_path), "{pkg_path} does not exist".format(**locals()) # By default the egg-links path are added after the pythonpaths. - # We need opposite behavior for ROS python (due to how devel workspace works) - # this is handle by both catkin_pip for ROS usage and pyros_setup for python usage. + # We need opposite behavior for ROS python (due to how devel workspace and underlays work) + # This is handle by both catkin_pip for ROS usage and pyros_setup for python usage. assert in_sys_path_ordered(*([pkg_path] + before)),\ "paths not appearing in sys.path in the expected order : {p}".format(p=([pkg_path] + before)) diff --git a/test/test_requirements.txt b/test/test_requirements.txt index 6090908..dfc6766 100644 --- a/test/test_requirements.txt +++ b/test/test_requirements.txt @@ -1 +1,2 @@ -cookiecutter \ No newline at end of file +cookiecutter +pyros-setup \ No newline at end of file diff --git a/travis_checks.bash b/travis_checks.bash index 62293a7..eb5aef0 100755 --- a/travis_checks.bash +++ b/travis_checks.bash @@ -38,6 +38,6 @@ elif [ "$ROS_FLOW" == "install" ]; then echo PYTHONPATH = $PYTHONPATH rospack profile # TMP disabling test from now, since mypippkg has no tests - #nosetests mypippkg - #python -m pytest --pyargs mypippkg + nosetests mypippkg + python -m pytest --pyargs mypippkg fi From 29ad71a6660d0c7cda29e0deeb81a2ff090e9ea9 Mon Sep 17 00:00:00 2001 From: AlexV Date: Tue, 21 Feb 2017 16:32:03 +0900 Subject: [PATCH 21/24] improved tests. added test for check with pyros_setup and interractive debugging --- test/CMakeLists.txt | 56 +++++++++-- test/catkin_pip_pytest/conftest.py | 15 ++- test/catkin_pip_pytest/test_easyinstallpth.py | 1 - test/catkin_pip_pytest/test_syspath.py | 99 ++++++++++++------- test/catkin_pip_pytest/test_syspath_pyros.py | 87 ++++++++++++++++ 5 files changed, 203 insertions(+), 55 deletions(-) create mode 100644 test/catkin_pip_pytest/test_syspath_pyros.py diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cad5324..42330c7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -104,14 +104,54 @@ add_subdirectory(pypackage-minimal) if (CATKIN_ENABLE_TESTING) - ######################## - # Nose Tests should work - ######################## - - # Testing environment setup by catkin_pip - # We need to add all subprojects as dependencies for this test - catkin_add_pytests( - catkin_pip_pytest + ######### + # Pytests + ######### + + # Testing one by one to make sure pyros_setup import doesnt influence test it shouldnt influence... + catkin_add_pytests(catkin_pip_pytest/test_easyinstallpth.py + --build-dir ${CMAKE_BINARY_DIR} + DEPENDENCIES + ${mypippkg_PIP_TARGET} # from pipproject + ${nameless_PIP_TARGET} # from pylibrary + ${python_boilerplate_PIP_TARGET} # from pypackage + ${cookiecutter_pypackage_minimal_PIP_TARGET} # from pypackage-minimal + ) + catkin_add_pytests(catkin_pip_pytest/test_path.py + --build-dir ${CMAKE_BINARY_DIR} + DEPENDENCIES + ${mypippkg_PIP_TARGET} # from pipproject + ${nameless_PIP_TARGET} # from pylibrary + ${python_boilerplate_PIP_TARGET} # from pypackage + ${cookiecutter_pypackage_minimal_PIP_TARGET} # from pypackage-minimal + ) + catkin_add_pytests(catkin_pip_pytest/test_site.py + --build-dir ${CMAKE_BINARY_DIR} + DEPENDENCIES + ${mypippkg_PIP_TARGET} # from pipproject + ${nameless_PIP_TARGET} # from pylibrary + ${python_boilerplate_PIP_TARGET} # from pypackage + ${cookiecutter_pypackage_minimal_PIP_TARGET} # from pypackage-minimal + ) + catkin_add_pytests(catkin_pip_pytest/test_import.py + --build-dir ${CMAKE_BINARY_DIR} + DEPENDENCIES + ${mypippkg_PIP_TARGET} # from pipproject + ${nameless_PIP_TARGET} # from pylibrary + ${python_boilerplate_PIP_TARGET} # from pypackage + ${cookiecutter_pypackage_minimal_PIP_TARGET} # from pypackage-minimal + ) + catkin_add_pytests(catkin_pip_pytest/test_syspath.py + --build-dir ${CMAKE_BINARY_DIR} + DEPENDENCIES + ${mypippkg_PIP_TARGET} # from pipproject + ${nameless_PIP_TARGET} # from pylibrary + ${python_boilerplate_PIP_TARGET} # from pypackage + ${cookiecutter_pypackage_minimal_PIP_TARGET} # from pypackage-minimal + ) + + # Because we need to validate that pyros fixes editable pkg path ordering for us + catkin_add_pytests(catkin_pip_pytest/test_syspath_pyros.py --build-dir ${CMAKE_BINARY_DIR} DEPENDENCIES ${mypippkg_PIP_TARGET} # from pipproject diff --git a/test/catkin_pip_pytest/conftest.py b/test/catkin_pip_pytest/conftest.py index b8e978f..1684738 100644 --- a/test/catkin_pip_pytest/conftest.py +++ b/test/catkin_pip_pytest/conftest.py @@ -10,24 +10,25 @@ def pytest_addoption(parser): def build_dir(request): return request.config.getoption("--build-dir") +# TODO : review these builddir fixtures... @pytest.fixture -def build_dir(request): +def build_dir(): # current dir should be the build directory (binary dir for cmake) # otherwise cmake tests commands are broken return os.getcwd() @pytest.fixture -def catkin_pip_env_dir(request): +def catkin_pip_env_dir(): # lets find catkin_pip_env space in current dir (cwd should be the build directory) # expected setup.sh - cpe = os.path.join(build_dir(request), 'catkin_pip_env') + cpe = os.path.join(build_dir(), 'catkin_pip_env') return cpe @pytest.fixture -def git_working_tree(request): +def git_working_tree(): mod_path_split = os.path.abspath(__file__).split(os.sep) # file should be absolute : we replace the first term @@ -49,7 +50,7 @@ def git_working_tree(request): @pytest.fixture -def devel_space(request): +def devel_space(): # lets find devel space in current dir (cwd should be the build directory) # expected setup.sh assert os.path.exists(os.path.abspath(os.path.join('devel', 'setup.sh'))), "devel/setup.bash not found in {0}".format(os.getcwd()) @@ -58,11 +59,9 @@ def devel_space(request): @pytest.fixture -def install_space(request): +def install_space(): # lets find devel space in current dir (cwd should be the build directory) # expected setup.sh assert os.path.exists(os.path.abspath(os.path.join('install', 'setup.sh'))), "install/setup.bash not found in {0}".format(os.getcwd()) ws = os.path.abspath('install') return ws - - diff --git a/test/catkin_pip_pytest/test_easyinstallpth.py b/test/catkin_pip_pytest/test_easyinstallpth.py index bd3fea1..3ceb2ca 100644 --- a/test/catkin_pip_pytest/test_easyinstallpth.py +++ b/test/catkin_pip_pytest/test_easyinstallpth.py @@ -2,7 +2,6 @@ from __future__ import print_function import os -import pytest def test_easyinstall_content(devel_space, git_working_tree): diff --git a/test/catkin_pip_pytest/test_syspath.py b/test/catkin_pip_pytest/test_syspath.py index 48534c3..2a0a8b3 100644 --- a/test/catkin_pip_pytest/test_syspath.py +++ b/test/catkin_pip_pytest/test_syspath.py @@ -2,23 +2,63 @@ from __future__ import print_function import os -import sys import pytest -import importlib +import sys + + +""" +This module is a test for catkin_pip. +It is expected to run FROM CMAKE with pytest, from the build directory +Ex : If you ran ./travis_checks.bash that is /home/alexv/Projects/ros_pyros_ws/src/catkin_pip/testbuild +""" def in_sys_path_ordered(*arg_paths): """ - asserts that path1 is before path2 in sys.path + Checks if arg_paths are listed int he same order as in sys.path :param arg_paths: the arguments are a list of path :return: True if all paths appear in sys.path, and in the arguments order. False otherwise. """ return all(p in sys.path for p in arg_paths) and \ - all([sys.path.index(p) < sys.path.index(q) for p, q in zip(arg_paths,arg_paths[1:])] + all([sys.path.index(p) < sys.path.index(q) for p, q in zip(arg_paths, arg_paths[1:])] if len(arg_paths) else True) -def test_devel_space_in_sys_path(devel_space): +# +# TESTS +# + +def test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir): + print(sys.path) + assert in_sys_path_ordered( + os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages'), + ), "{site_pkgs} not appearing in sys.path".format( + site_pkgs=os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages') + ) + + +def test_devel_site_in_sys_path(devel_space): + print(sys.path) + assert in_sys_path_ordered( + os.path.join(devel_space, 'lib/python2.7/site-packages'), + ), "{site_pkgs} not in sys.path".format( + site_pkgs=os.path.join(devel_space, 'lib/python2.7/site-packages') + ) + + +@pytest.mark.xfail(strict=True, reason="dist-packages does not exist in our example since all packages use catkin_pip") +def test_devel_dist_in_sys_path(devel_space): + print(sys.path) + assert in_sys_path_ordered( + os.path.join(devel_space, 'lib/python2.7/dist-packages'), + ), "{dist_pkgs} in sys.path".format( + dist_pkgs=os.path.join(devel_space, 'lib/python2.7/dist-packages') + ) + 1/0 + + +# this should pass only if started from cmake, ie. the ROS way via catkin_pip +def test_devel_site_before_devel_dist_in_sys_path(devel_space): print(sys.path) # Verifying relative path order @@ -31,47 +71,30 @@ def test_devel_space_in_sys_path(devel_space): ) -def test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir): +def test_devel_site_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space): print(sys.path) - - # Verifying relative path order assert in_sys_path_ordered( + os.path.join(devel_space, 'lib/python2.7/site-packages'), os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages'), - ), "{site_pkgs} not appearing in sys.path".format( - site_pkgs=os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages') + ), "{site_pkgs} not appearing before {catkin_pkgs} in sys.path".format( + site_pkgs=os.path.join(devel_space, 'lib/python2.7/site-packages'), + catkin_pkgs=os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages') ) -def test_sys_path_editable(git_working_tree, devel_space): +@pytest.mark.xfail(strict=True, reason="dist-packages does not exist in our example since all packages use catkin_pip") +def test_devel_dist_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space): print(sys.path) + assert in_sys_path_ordered( + os.path.join(devel_space, 'lib/python2.7/dist-packages'), + os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages'), + ), "{dist_pkgs} not appearing before {catkin_pkgs} in sys.path".format( + dist_pkgs=os.path.join(devel_space, 'lib/python2.7/dist-packages'), + catkin_pkgs=os.path.join(catkin_pip_env_dir, 'lib/python2.7/site-packages') + ) - # IMPORTANT : Without pyros_setup configuration activated, - # the develop packages are set in sys.path AFTER the site-packages and dist-packages - # This is because of incompatibilities between ROS and python ways of handling PYTHONPATH. - # It is usually not a problem, unless : - # - you are working in a devel space with catkin_pip and expect to use a editable package from there - # - you have the same package installed somewhere else in the PYTHONPATH - # And in that case it might end up being *before* the source dir in sys.path - # - # => Without pyros_setup, the installed version will be preferred - # This is the python (and tools) way of working, since PYTHONPATH purpose is to override file-based setup, - # like editable packages. - # - # => With pyros_setup, the devel version will be preferred - # This is the ROS way of working, since PYTHONPATH order determine the order of folder when looking for a package. - # - # Both will work fine in most cases, but one might want to keep this corner case in mind... - - import pyros_setup - - # We need to pass the proper workspace here - pyros_setup.configurable_import().configure({ - 'WORKSPACES': [ - devel_space - ], # This should be the same as pytest devel_space fixture - 'DISTRO': pyros_setup.DETECTED_DISTRO, - }).activate() +def test_sys_path_editable(git_working_tree, devel_space): print(sys.path) def pkg_path_in_sys_path(pkg_path, before=[]): @@ -96,5 +119,5 @@ def pkg_path_in_sys_path(pkg_path, before=[]): pkg_path_in_sys_path(ss, [ os.path.join(devel_space, 'lib/python2.7/site-packages'), - os.path.join(devel_space, 'lib/python2.7/dist-packages') + # os.path.join(devel_space, 'lib/python2.7/dist-packages') # no dist-packages folder since all our examples are using catkin_pip ]) diff --git a/test/catkin_pip_pytest/test_syspath_pyros.py b/test/catkin_pip_pytest/test_syspath_pyros.py new file mode 100644 index 0000000..478a0fb --- /dev/null +++ b/test/catkin_pip_pytest/test_syspath_pyros.py @@ -0,0 +1,87 @@ +from __future__ import absolute_import +from __future__ import print_function + +import pytest + +""" +This module is a test for catkin_pip, with pyros_setup. +It is expected to run DIRECTLY with pytest, from the build directory +Ex : If you ran ./travis_checks.bash that is /home/alexv/Projects/ros_pyros_ws/src/catkin_pip/testbuild + +It is here to make testing/debugging sys.path issues from IDE easier : +- run ./travis_checks +- setup a test target (careful with the working directory) +- run it (and you can debug inside...) +""" + +from . import test_syspath +from . import conftest + +# IMPORTANT : Without pyros_setup configuration activated, +# the develop packages are set in sys.path AFTER the site-packages and dist-packages +# This is because of incompatibilities between ROS and python ways of handling PYTHONPATH. +# It is usually not a problem, unless : +# - you are working in a devel space with catkin_pip and expect to use a editable package from there +# - you have the same package installed somewhere else in the PYTHONPATH +# And in that case it might end up being *before* the source dir in sys.path +# +# => Without pyros_setup, the installed version will be preferred +# This is the python (and tools) way of working, since PYTHONPATH purpose is to override file-based setup, +# like editable packages. +# +# => With pyros_setup, the devel version will be preferred +# This is the ROS way of working, since PYTHONPATH order determine the order of folder when looking for a package. +# +# Both will work fine in most cases, but one might want to keep this corner case in mind... + +import pyros_setup + +# Getting devel_space path using fixture by +devel_space = conftest.devel_space() + +# We need to pass the proper workspace here +pyros_setup.configurable_import().configure({ + 'WORKSPACES': [ + devel_space + ], # This should be the same as pytest devel_space fixture + 'DISTRO': pyros_setup.DETECTED_DISTRO, +}).activate() + + +# +# Tests after importing pyros_setup and automatically setting configuration +# All these should PASS in any case. +# + + +# this test would have no meaning with pyros_setup, since pyros_setup does not rely and has no knowledge of catkin_pip +# def test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir): +# return test_syspath.test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir) + + +def test_devel_site_in_sys_path(devel_space): + return test_syspath.test_devel_site_in_sys_path(devel_space) + + +# this fails since dist-packages does not exists +@pytest.mark.xfail(reason="dist-packages does not exist in our example since all packages use catkin_pip") +def test_devel_dist_in_sys_path(devel_space): + return test_syspath.test_devel_dist_in_sys_path(devel_space) + + +# this fails since dist-packages does not exists +@pytest.mark.xfail(strict=True, reason="dist-packages does not exist in our example since all packages use catkin_pip") +def test_devel_site_before_devel_dist_in_sys_path(devel_space): + return test_syspath.test_devel_site_before_devel_dist_in_sys_path(devel_space) + +# this test would have no meaning with pyros_setup, since pyros_setup does not rely and has no knowledge of catkin_pip +# def test_devel_site_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space): +# return test_syspath.test_devel_site_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space) + +# this test would have no meaning with pyros_setup, since pyros_setup does not rely and has no knowledge of catkin_pip +# def test_devel_dist_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space): +# return test_syspath.test_devel_dist_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space) + + +def test_sys_path_editable(git_working_tree, devel_space): + return test_syspath.test_sys_path_editable(git_working_tree, devel_space) From 6b78e49c25e90381d6f57d7fa91f824e3283736a Mon Sep 17 00:00:00 2001 From: AlexV Date: Tue, 21 Feb 2017 20:22:37 +0900 Subject: [PATCH 22/24] envhooks now not adding to pythonpath from pth files. too confusing. envhooks now not adding non existent paths to pythonpath. maybe good idea to match site module behavior. fixing tests. --- .../42.site_packages.sh.develspace.in | 52 ++++++++------ cmake/scripts/path_prepend.sh | 6 +- test/CMakeLists.txt | 7 +- test/catkin_pip_pytest/conftest.py | 44 ++++++++++-- test/catkin_pip_pytest/test_syspath.py | 3 +- test/catkin_pip_pytest/test_syspath_pyros.py | 68 ++++++------------- test/pypackage-minimal/CMakeLists.txt | 8 +-- test/test_requirements.txt | 2 +- 8 files changed, 106 insertions(+), 84 deletions(-) diff --git a/cmake/env-hooks/42.site_packages.sh.develspace.in b/cmake/env-hooks/42.site_packages.sh.develspace.in index 46d0dbb..34656f0 100755 --- a/cmake/env-hooks/42.site_packages.sh.develspace.in +++ b/cmake/env-hooks/42.site_packages.sh.develspace.in @@ -19,38 +19,50 @@ set -o noglob # Setting up the catkin environment itself # This needs to be in a different prefix, to be able to generate an install dir with setuptools, yet clear of any catkin_pip artifacts. -#echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails +if [ -d "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" ]; then + #echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" + PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails +fi # Adding bin/ into the path -#echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@ in front of PATH" -PATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@" "${PATH}") || ${PATH} # to be safe if path_prepend fails +if [ -d "@CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@" ]; then + #echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@ in front of PATH" + PATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@" "${PATH}") || ${PATH} # to be safe if path_prepend fails +fi # Somehow It seems we need to also (re)add dist-packages here to get package tests to work with ROS packages in same workspace # Especially when running tests with catkin_pip from source... there seem to be some workflow issue due to catkin cache environment... # Anyway it doesnt hurt to have it here as well. # this is usual for catkin -> no echo. This might actually be a fix for catkin not setting this before running tests. -#echo "Inserting @CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails -# Problem : before which workspace ? we might break workspace overlay order here + +if [ -d "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@" ]; then + #echo "Inserting @CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" + PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails + # Problem : before which workspace ? we might break workspace overlay order here +fi # We combine our build path (catkin_pip_env) with site-packages (pip default) location, before attempting to prepend to the python path # Careful : echo is only possible if we are in a bash script (not used by env.sh) -#echo "Inserting @CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ before @CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@ into PYTHONPATH" -PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@") || ${PYTHONPATH} # to be safe if path_prepend fails - -# Prepending here our easy-install.pth content into PYTHONPATH, for overlayed workspace to get it before the workspace path -# Somehow Python pth/.egg behavior doesnt work well with workspace overlays, based on PYTHONPATH. -# However be aware of : https://github.com/pypa/pip/issues/3160 -# And : https://github.com/pypa/pip/issues/4261 -if [ -f "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" ]; then - # Ref : http://stackoverflow.com/questions/4165135/how-to-use-while-read-bash-to-read-the-last-line-in-a-file-if-there-s-no-new - while IFS='' read -r DEVEL_PKG_PATH || [ -n "$DEVEL_PKG_PATH" ]; do - #echo "Inserting $DEVEL_PKG_PATH before "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" into PYTHONPATH" - PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "$DEVEL_PKG_PATH" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") || ${PYTHONPATH} # to be safe if path_prepend fails - done <"@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" + +if [ -d "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" ]; then + #echo "Inserting @CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ before @CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@ into PYTHONPATH" + PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PYTHON_INSTALL_DIR@") || ${PYTHONPATH} # to be safe if path_prepend fails fi +# NOT DOING THIS any longer : it half-works but somehow doesnt change behavior when using catkin cmake scripts : usual python way, editable links are after python path +# TO FIX IT : use pyros_setup +# # Prepending here our easy-install.pth content into PYTHONPATH, for overlayed workspace to get it before the workspace path +# # Somehow Python pth/.egg behavior doesnt work well with workspace overlays, based on PYTHONPATH. +# # However be aware of : https://github.com/pypa/pip/issues/3160 +# # And : https://github.com/pypa/pip/issues/4261 +# if [ -f "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" ]; then +# # Ref : http://stackoverflow.com/questions/4165135/how-to-use-while-read-bash-to-read-the-last-line-in-a-file-if-there-s-no-new +# while IFS='' read -r DEVEL_PKG_PATH || [ -n "$DEVEL_PKG_PATH" ]; do +# #echo "Inserting $DEVEL_PKG_PATH before "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" into PYTHONPATH" +# PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "$DEVEL_PKG_PATH" "${PYTHONPATH}" "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") || ${PYTHONPATH} # to be safe if path_prepend fails +# done <"@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@/easy-install.pth" +# fi + # Exporting is required to make sure we get the new value in children processes (catkin_make, test runs, etc.) export PYTHONPATH diff --git a/cmake/scripts/path_prepend.sh b/cmake/scripts/path_prepend.sh index 9719b3f..dd2d74a 100755 --- a/cmake/scripts/path_prepend.sh +++ b/cmake/scripts/path_prepend.sh @@ -32,7 +32,7 @@ if [ $# -eq 3 ]; then fi # if PATH is empty -if [ -z "$LIST" -o ]; then +if [ -z "$LIST" ]; then LIST="$ARG" else # we move it from tail to head of paths in PATH list @@ -40,7 +40,7 @@ else LIST="" for p in $PATH_LIST; do if [ X"$p" = X"${BEFORE:-}" ]; then - #echo "BEFORE FOUND" + BEFORE_FOUND=1 # to mark we have found BEFORE [ -z "${LIST:-}" ] && LIST=${ARG} || LIST=${LIST}:${ARG} #echo $LIST fi @@ -50,7 +50,7 @@ else fi done # we prepend ARG here, after we made sure it is not in LIST any longer - if [ -z ${BEFORE:-} ]; then + if [ -z "${BEFORE:-}" -o -z "${BEFORE_FOUND:-}" ]; then # if there was no before or if it was not found, we put it in front [ -z "${LIST:-}" ] && LIST="$ARG" || LIST="$ARG":"${LIST}" fi fi diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 42330c7..6ecfb74 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -34,8 +34,7 @@ endif() if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pipproject/mypippkg") message(STATUS " ... Getting cookiecutter-pipproject template from cookiecutter ...") execute_process( - # TMP waiting for PR - COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/asmodehn/cookiecutter-pipproject.git --checkout adding_test_sample + COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/wdm0006/cookiecutter-pipproject.git WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pipproject RESULT_VARIABLE PIP_RESULT OUTPUT_VARIABLE PIP_VARIABLE @@ -69,7 +68,7 @@ if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/pypackage-minimal/cookiecutter_pypac message(STATUS " ... Getting cookiecutter-pypackage-minimal template from cookiecutter ...") execute_process( # TMP : fork to get proper default package name. upstream : https://github.com/kragniz/cookiecutter-pypackage-minimal - COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/asmodehn/cookiecutter-pypackage-minimal + COMMAND ${CATKIN_ENV} ${CATKIN_COOKIECUTTER} --no-input https://github.com/asmodehn/cookiecutter-pypackage-minimal --checkout adding_test WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/pypackage-minimal RESULT_VARIABLE PIP_RESULT OUTPUT_VARIABLE PIP_VARIABLE @@ -152,7 +151,7 @@ if (CATKIN_ENABLE_TESTING) # Because we need to validate that pyros fixes editable pkg path ordering for us catkin_add_pytests(catkin_pip_pytest/test_syspath_pyros.py - --build-dir ${CMAKE_BINARY_DIR} + --capture=no --build-dir ${CMAKE_BINARY_DIR} DEPENDENCIES ${mypippkg_PIP_TARGET} # from pipproject ${nameless_PIP_TARGET} # from pylibrary diff --git a/test/catkin_pip_pytest/conftest.py b/test/catkin_pip_pytest/conftest.py index 1684738..feed551 100644 --- a/test/catkin_pip_pytest/conftest.py +++ b/test/catkin_pip_pytest/conftest.py @@ -1,6 +1,10 @@ import os +import sys import pytest +# Imports CANNOT be in fixtues (pytest would cleanup modules, preventing re-import...) +import pyros_setup + def pytest_addoption(parser): parser.addoption("--build-dir", action="store", default="build", help="the build directory") @@ -10,25 +14,24 @@ def pytest_addoption(parser): def build_dir(request): return request.config.getoption("--build-dir") -# TODO : review these builddir fixtures... @pytest.fixture -def build_dir(): +def build_dir(request): # current dir should be the build directory (binary dir for cmake) # otherwise cmake tests commands are broken return os.getcwd() @pytest.fixture -def catkin_pip_env_dir(): +def catkin_pip_env_dir(request): # lets find catkin_pip_env space in current dir (cwd should be the build directory) # expected setup.sh - cpe = os.path.join(build_dir(), 'catkin_pip_env') + cpe = os.path.join(build_dir(request), 'catkin_pip_env') return cpe @pytest.fixture -def git_working_tree(): +def git_working_tree(request): mod_path_split = os.path.abspath(__file__).split(os.sep) # file should be absolute : we replace the first term @@ -65,3 +68,34 @@ def install_space(): assert os.path.exists(os.path.abspath(os.path.join('install', 'setup.sh'))), "install/setup.bash not found in {0}".format(os.getcwd()) ws = os.path.abspath('install') return ws + +@pytest.fixture +def pyros(): + # IMPORTANT : Without pyros_setup configuration activated, + # the develop packages are set in sys.path AFTER the site-packages and dist-packages + # This is because of incompatibilities between ROS and python ways of handling PYTHONPATH. + # It is usually not a problem, unless : + # - you are working in a devel space with catkin_pip and expect to use a editable package from there + # - you have the same package installed somewhere else in the PYTHONPATH + # And in that case it might end up being *before* the source dir in sys.path + # + # => Without pyros_setup, the installed version will be preferred + # This is the python (and tools) way of working, since PYTHONPATH purpose is to override file-based setup, + # like editable packages. + # + # => With pyros_setup, the devel version will be preferred + # This is the ROS way of working, since PYTHONPATH order determine the order of folder when looking for a package. + # + # Both will work fine in most cases, but one might want to keep this corner case in mind... + + print("sys.path before pyros_setup {pyros_setup.__file__} :\n{sys.path}".format(**globals())) + + # We need to pass the proper workspace here + pyros_setup.configurable_import().configure({ + 'WORKSPACES': [ + devel_space() + ], # This should be the same as pytest devel_space fixture + 'DISTRO': pyros_setup.DETECTED_DISTRO, + }).activate() + + # We cannot move pyros_setup outside the fixture (pytest would clean it up and break module imports) diff --git a/test/catkin_pip_pytest/test_syspath.py b/test/catkin_pip_pytest/test_syspath.py index 2a0a8b3..419b1ea 100644 --- a/test/catkin_pip_pytest/test_syspath.py +++ b/test/catkin_pip_pytest/test_syspath.py @@ -57,7 +57,7 @@ def test_devel_dist_in_sys_path(devel_space): 1/0 -# this should pass only if started from cmake, ie. the ROS way via catkin_pip +@pytest.mark.xfail(strict=True, reason="dist-packages does not exist in our example since all packages use catkin_pip") def test_devel_site_before_devel_dist_in_sys_path(devel_space): print(sys.path) @@ -94,6 +94,7 @@ def test_devel_dist_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel ) +@pytest.mark.xfail(strict=True, reason="With usual python and catkin cmake scripts, editable links are added so sys.path after pythonpath.") def test_sys_path_editable(git_working_tree, devel_space): print(sys.path) diff --git a/test/catkin_pip_pytest/test_syspath_pyros.py b/test/catkin_pip_pytest/test_syspath_pyros.py index 478a0fb..ac486e0 100644 --- a/test/catkin_pip_pytest/test_syspath_pyros.py +++ b/test/catkin_pip_pytest/test_syspath_pyros.py @@ -2,6 +2,7 @@ from __future__ import print_function import pytest +import sys """ This module is a test for catkin_pip, with pyros_setup. @@ -12,76 +13,51 @@ - run ./travis_checks - setup a test target (careful with the working directory) - run it (and you can debug inside...) + +Note however that this test module is run with catkin_pip tests as well +to make sure pyros_setup fixes our catkin import problems. """ from . import test_syspath -from . import conftest - -# IMPORTANT : Without pyros_setup configuration activated, -# the develop packages are set in sys.path AFTER the site-packages and dist-packages -# This is because of incompatibilities between ROS and python ways of handling PYTHONPATH. -# It is usually not a problem, unless : -# - you are working in a devel space with catkin_pip and expect to use a editable package from there -# - you have the same package installed somewhere else in the PYTHONPATH -# And in that case it might end up being *before* the source dir in sys.path -# -# => Without pyros_setup, the installed version will be preferred -# This is the python (and tools) way of working, since PYTHONPATH purpose is to override file-based setup, -# like editable packages. -# -# => With pyros_setup, the devel version will be preferred -# This is the ROS way of working, since PYTHONPATH order determine the order of folder when looking for a package. -# -# Both will work fine in most cases, but one might want to keep this corner case in mind... - -import pyros_setup - -# Getting devel_space path using fixture by -devel_space = conftest.devel_space() - -# We need to pass the proper workspace here -pyros_setup.configurable_import().configure({ - 'WORKSPACES': [ - devel_space - ], # This should be the same as pytest devel_space fixture - 'DISTRO': pyros_setup.DETECTED_DISTRO, -}).activate() - # -# Tests after importing pyros_setup and automatically setting configuration +# Tests importing pyros_setup via fixture and automatically setting configuration # All these should PASS in any case. # -# this test would have no meaning with pyros_setup, since pyros_setup does not rely and has no knowledge of catkin_pip -# def test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir): -# return test_syspath.test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir) +@pytest.mark.xfail(reason="this has no meaning without catkin_pip, since pyros_setup does not rely and has no knowledge of catkin_pip") +def test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir): + return test_syspath.test_catkin_pip_env_dir_in_sys_path(catkin_pip_env_dir) -def test_devel_site_in_sys_path(devel_space): +def test_devel_site_in_sys_path(pyros, devel_space): return test_syspath.test_devel_site_in_sys_path(devel_space) # this fails since dist-packages does not exists @pytest.mark.xfail(reason="dist-packages does not exist in our example since all packages use catkin_pip") -def test_devel_dist_in_sys_path(devel_space): +def test_devel_dist_in_sys_path(pyros, devel_space): return test_syspath.test_devel_dist_in_sys_path(devel_space) # this fails since dist-packages does not exists @pytest.mark.xfail(strict=True, reason="dist-packages does not exist in our example since all packages use catkin_pip") -def test_devel_site_before_devel_dist_in_sys_path(devel_space): +def test_devel_site_before_devel_dist_in_sys_path(pyros, devel_space): return test_syspath.test_devel_site_before_devel_dist_in_sys_path(devel_space) -# this test would have no meaning with pyros_setup, since pyros_setup does not rely and has no knowledge of catkin_pip -# def test_devel_site_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space): -# return test_syspath.test_devel_site_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space) -# this test would have no meaning with pyros_setup, since pyros_setup does not rely and has no knowledge of catkin_pip -# def test_devel_dist_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space): -# return test_syspath.test_devel_dist_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space) +@pytest.mark.xfail(reason="this has no meaning without catkin_pip, since pyros_setup does not rely and has no knowledge of catkin_pip") +def test_devel_site_before_catkin_pip_site_in_sys_path(pyros, catkin_pip_env_dir, devel_space): + return test_syspath.test_devel_site_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space) + + +@pytest.mark.xfail(reason="this has no meaning without catkin_pip, since pyros_setup does not rely and has no knowledge of catkin_pip") +def test_devel_dist_before_catkin_pip_site_in_sys_path(pyros, catkin_pip_env_dir, devel_space): + return test_syspath.test_devel_dist_before_catkin_pip_site_in_sys_path(catkin_pip_env_dir, devel_space) -def test_sys_path_editable(git_working_tree, devel_space): +# Using Pyros_setup is the only way to get this to work as expected. +# That is getting devel packages in sys.path before ROS workspaces from PYTHONPATH. +def test_sys_path_editable(pyros, git_working_tree, devel_space): return test_syspath.test_sys_path_editable(git_working_tree, devel_space) diff --git a/test/pypackage-minimal/CMakeLists.txt b/test/pypackage-minimal/CMakeLists.txt index 756d9b6..88f028e 100644 --- a/test/pypackage-minimal/CMakeLists.txt +++ b/test/pypackage-minimal/CMakeLists.txt @@ -19,7 +19,7 @@ catkin_pip_package(cookiecutter_pypackage_minimal ${PIP_PROJECT_DIR}) # CAREFUL : all projects for test here will share the same workspace. pip might have conflicts... ## Unit tests -# if (CATKIN_ENABLE_TESTING) -# catkin_add_nosetests(${PIP_PROJECT_DIR}/tests DEPENDENCIES cookiecutter_pypackage_minimal) -# catkin_add_pytests(${PIP_PROJECT_DIR}/tests DEPENDENCIES cookiecutter_pypackage_minimal) -# endif() +if (CATKIN_ENABLE_TESTING) + catkin_add_nosetests(${PIP_PROJECT_DIR}/tests DEPENDENCIES cookiecutter_pypackage_minimal) + catkin_add_pytests(${PIP_PROJECT_DIR}/tests DEPENDENCIES cookiecutter_pypackage_minimal) +endif() diff --git a/test/test_requirements.txt b/test/test_requirements.txt index dfc6766..9543920 100644 --- a/test/test_requirements.txt +++ b/test/test_requirements.txt @@ -1,2 +1,2 @@ cookiecutter -pyros-setup \ No newline at end of file +pyros-setup>=0.2.0 \ No newline at end of file From 5ef3949fe98db6007eba685371e34ab7806b67fc Mon Sep 17 00:00:00 2001 From: AlexV Date: Tue, 21 Feb 2017 20:50:24 +0900 Subject: [PATCH 23/24] always adding the catkin_pip_env path to pythonpath to be able to find basic tools from the first time around. --- cmake/env-hooks/42.site_packages.sh.develspace.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/env-hooks/42.site_packages.sh.develspace.in b/cmake/env-hooks/42.site_packages.sh.develspace.in index 34656f0..bab0c16 100755 --- a/cmake/env-hooks/42.site_packages.sh.develspace.in +++ b/cmake/env-hooks/42.site_packages.sh.develspace.in @@ -19,10 +19,10 @@ set -o noglob # Setting up the catkin environment itself # This needs to be in a different prefix, to be able to generate an install dir with setuptools, yet clear of any catkin_pip artifacts. -if [ -d "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" ]; then - #echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" - PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails -fi +# Also this path needs to ALWAYS be added to PYTHONPATH. Mandatory the first time we run catkin_pip to setup the environment. +#echo "Inserting @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ in front of PYTHONPATH" +PYTHONPATH=$("@CATKIN_PIP_SCRIPTS_PATH@/path_prepend.sh" "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@" "${PYTHONPATH}") || ${PYTHONPATH} # to be safe if path_prepend fails + # Adding bin/ into the path if [ -d "@CATKIN_PIP_ENV@/@CATKIN_GLOBAL_BIN_DESTINATION@" ]; then From 8774c93c3e1b5ecd4dafe09eac2d560f095762fd Mon Sep 17 00:00:00 2001 From: AlexV Date: Tue, 21 Feb 2017 20:54:08 +0900 Subject: [PATCH 24/24] commenting install tests... we dont have any. --- travis_checks.bash | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/travis_checks.bash b/travis_checks.bash index eb5aef0..c747cd0 100755 --- a/travis_checks.bash +++ b/travis_checks.bash @@ -37,7 +37,6 @@ elif [ "$ROS_FLOW" == "install" ]; then source install/setup.bash echo PYTHONPATH = $PYTHONPATH rospack profile - # TMP disabling test from now, since mypippkg has no tests - nosetests mypippkg - python -m pytest --pyargs mypippkg + # No tests embedded in installed version of catkin_pip + # We probably want to do some clever prerelease here... fi