From 773fa5aeabea1f43b8c2b4808771070e9b45b5e1 Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 9 Aug 2016 14:39:50 +0900 Subject: [PATCH 1/2] separating catkin_pip environment with workspace environment. added envhook for loading caktin_pip env on installspace. removing install script for python on windows for now (outdated). --- CMakeLists.txt | 6 ++-- cmake/catkin-pip-env.cmake.in | 5 ---- cmake/catkin-pip-prefix.cmake.in | 4 --- cmake/catkin-pip-setup.cmake.in | 1 + cmake/catkin-pip.cmake.in | 9 ++++-- .../42.site_packages.bash.develspace.in | 10 +++++-- .../42.site_packages.bash.installspace.in | 29 +++++++++++++++++++ cmake/nosetests.cmake.in | 2 +- cmake/pytest.cmake.in | 2 +- .../python_setuptools_install.bat.in | 17 ----------- .../templates/python_setuptools_install.sh.in | 20 +++++++++---- 11 files changed, 65 insertions(+), 40 deletions(-) create mode 100755 cmake/env-hooks/42.site_packages.bash.installspace.in delete mode 100644 cmake/templates/python_setuptools_install.bat.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 37d578b..02d8112 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,8 @@ catkin_package( configure_file(cmake/catkin-pip-base.req ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/catkin-pip-base.req COPYONLY) configure_file(cmake/catkin-pip-fixups.req ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/catkin-pip-fixups.req COPYONLY) configure_file(cmake/env-hooks/42.site_packages.bash.develspace.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks/42.site_packages.bash.develspace.in COPYONLY) -configure_file(cmake/templates/python_setuptools_install.bat.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/templates/python_setuptools_install.bat.in COPYONLY) +configure_file(cmake/env-hooks/42.site_packages.bash.installspace.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks/42.site_packages.bash.installspace.in COPYONLY) +#configure_file(cmake/templates/python_setuptools_install.bat.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/templates/python_setuptools_install.bat.in COPYONLY) configure_file(cmake/templates/python_setuptools_install.sh.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/templates/python_setuptools_install.sh.in COPYONLY) # to be found in install space and from package @@ -52,12 +53,13 @@ install(PROGRAMS # We install in cmake folder the files that are needed by cmake code install(FILES cmake/env-hooks/42.site_packages.bash.develspace.in + cmake/env-hooks/42.site_packages.bash.installspace.in DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks ) install(PROGRAMS - cmake/templates/python_setuptools_install.bat.in +# cmake/templates/python_setuptools_install.bat.in cmake/templates/python_setuptools_install.sh.in DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/templates diff --git a/cmake/catkin-pip-env.cmake.in b/cmake/catkin-pip-env.cmake.in index ad3cb9e..f775448 100644 --- a/cmake/catkin-pip-env.cmake.in +++ b/cmake/catkin-pip-env.cmake.in @@ -23,11 +23,6 @@ macro(catkin_pip_check_env config_prefix) "Quick patch will be applied to the current cmake process environment." ) - if ( FOUND_PPATH LESS 0 ) - message(STATUS "Adding ${config_prefix}/@CATKIN_GLOBAL_PYTHON_DESTINATION@ to PYTHONPATH") - set(ENV{PYTHONPATH} "${config_prefix}/@CATKIN_GLOBAL_PYTHON_DESTINATION@:$ENV{PYTHONPATH}") - endif() - if ( FOUND_PPPATH LESS 0 ) message(STATUS "Adding ${config_prefix}/@CATKIN_PIP_GLOBAL_PYTHON_DESTINATION@ to PYTHONPATH") set(ENV{PYTHONPATH} "${config_prefix}/@CATKIN_PIP_GLOBAL_PYTHON_DESTINATION@:$ENV{PYTHONPATH}") diff --git a/cmake/catkin-pip-prefix.cmake.in b/cmake/catkin-pip-prefix.cmake.in index 7a1a3a5..559c90e 100644 --- a/cmake/catkin-pip-prefix.cmake.in +++ b/cmake/catkin-pip-prefix.cmake.in @@ -49,10 +49,6 @@ macro(catkin_pip_setup_prefix ws_prefix) message( FATAL_ERROR "pip system command not found. Make sure you have installed the python-pip package on your system.") endif() - # message(STATUS " ... Creating pip packages prefix ...") - # If needed we create the directory (to avoid later errors) - # file(MAKE_DIRECTORY ${ws_prefix}/@CATKIN_PIP_GLOBAL_PYTHON_DESTINATION@) - message(STATUS " ... Retrieving catkin_pip requirements using system pip ...") # We need to find a pip command that works for old pip versions (indigo supports trusty which is pip v1.5.4) diff --git a/cmake/catkin-pip-setup.cmake.in b/cmake/catkin-pip-setup.cmake.in index cd2bd80..bd6a0ae 100644 --- a/cmake/catkin-pip-setup.cmake.in +++ b/cmake/catkin-pip-setup.cmake.in @@ -37,6 +37,7 @@ function(catkin_pip_python_setup) # assert(PYTHON_INSTALL_DIR) + assert(CATKIN_PIP_PYTHON_INSTALL_DIR) set(INSTALL_CMD_WORKING_DIRECTORY ${package_path}) if(NOT WIN32) diff --git a/cmake/catkin-pip.cmake.in b/cmake/catkin-pip.cmake.in index d99732f..0f0deb0 100644 --- a/cmake/catkin-pip.cmake.in +++ b/cmake/catkin-pip.cmake.in @@ -27,12 +27,17 @@ if ( NOT CATKIN_PIP_ENV_HOOKS_PATH ) endif() # Since we need the envhook as soon as a package is using catkin-pip from source, but only in devel space -catkin_add_env_hooks(42.site_packages SHELLS bash DIRECTORY ${CATKIN_PIP_ENV_HOOKS_PATH} SKIP_INSTALL) +catkin_add_env_hooks(42.site_packages SHELLS bash DIRECTORY ${CATKIN_PIP_ENV_HOOKS_PATH}) # TODO : cmake should include this file only once, so the env hook is not recreated for every package that includes it... -catkin_pip_setup_prefix(${CATKIN_DEVEL_PREFIX}) +# We want the catkin environment to be used for both devel and install workspace +# to use latest versions python tools in both. +set(CATKIN_PIP_ENV ${CMAKE_BINARY_DIR}/catkin_pip_env CACHE PATH "The path containing the python environment for catkin_pip dependencies" ) +set(CATKIN_PIP_PYTHON_INSTALL_DIR @CATKIN_PIP_GLOBAL_PYTHON_DESTINATION@) +catkin_pip_setup_prefix(${CATKIN_PIP_ENV}) # # Install Space should have same behavior as package build # We want to check dependencies are all satisfied, without relying on pip requirements. +# But we want to use latest version python libraries (especially setuptools) # diff --git a/cmake/env-hooks/42.site_packages.bash.develspace.in b/cmake/env-hooks/42.site_packages.bash.develspace.in index 022455a..df0b51b 100755 --- a/cmake/env-hooks/42.site_packages.bash.develspace.in +++ b/cmake/env-hooks/42.site_packages.bash.develspace.in @@ -20,10 +20,16 @@ CATKIN_PIP_SCRIPTS="$(rospack find catkin_pip)/scripts" echo "Catkin-pip scripts found in $CATKIN_PIP_SCRIPTS" +#We need first the catkin_pip_env +# Careful : echo is only possible if we are a bash script (not used by env.sh) +echo "Prepending @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ to PYTHONPATH" +PYTHONPATH=$("${CATKIN_PIP_SCRIPTS}"/pythonpath_prepend.bash "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") +# TODO : maybe we should to that in .sh envhook, so that env.sh would set it up before any command we want to run (Check CATKIN_ENV doc)... + # We combine it with site-packages (pip default) location, before attempting to prepend to the python path # Careful : echo is only possible if we are a bash script (not used by env.sh) -echo "Prepending @CATKIN_DEVEL_PREFIX@/lib/python2.7/site-packages to PYTHONPATH" -PYTHONPATH=$("${CATKIN_PIP_SCRIPTS}"/pythonpath_prepend.bash "@CATKIN_DEVEL_PREFIX@/lib/python2.7/site-packages") +echo "Prepending @CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ to PYTHONPATH" +PYTHONPATH=$("${CATKIN_PIP_SCRIPTS}"/pythonpath_prepend.bash "@CATKIN_DEVEL_PREFIX@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") # to avoid side effect with multiple workspace having different version of catkin_pip unset CATKIN_PIP_SCRIPTS diff --git a/cmake/env-hooks/42.site_packages.bash.installspace.in b/cmake/env-hooks/42.site_packages.bash.installspace.in new file mode 100755 index 0000000..648d8e8 --- /dev/null +++ b/cmake/env-hooks/42.site_packages.bash.installspace.in @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Careful : env.sh *executes* it during cmake configure, but setup.sh source it ! +# If using a sh script, this can make a lot of issues... +# Reference to address these from a Bourne shell http://stackoverflow.com/a/29835459 + +# For catkin it is simpler to do add site-packages to PYTHONPATH in _setup_util.py.in than us doing it from here... +# but until catkin does this by default, the proper way from outside catkin is probably a env-hook + +##Documentation: +# The colon command simply has its arguments evaluated and then succeeds. +# It is the original shell comment notation (before '#' to end of line). For a long time, Bourne shell scripts had a colon as the first character. +# The C Shell would read a script and use the first character to determine whether it was for the C Shell (a '#' hash) or the Bourne shell (a ':' colon). +# Then the kernel got in on the act and added support for '#!/path/to/program' and the Bourne shell got '#' comments, and the colon convention went by the wayside. +# But if you come across a script that starts with a colon (Like this one), now you will know why. ~ Jonathan Leffler + +# We should enable site-package only on devel space. +# Install space should behave like packages, everything in debian layout. + +# finding our useful script from catkin_pip using rospack (work from source or bin package) +CATKIN_PIP_SCRIPTS="$(rospack find catkin_pip)/scripts" +echo "Catkin-pip scripts found in $CATKIN_PIP_SCRIPTS" + +# We need ONLY the catkin_pip_env +# Careful : echo is only possible if we are a bash script (not used by env.sh) +echo "Prepending @CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@ to PYTHONPATH" +PYTHONPATH=$("${CATKIN_PIP_SCRIPTS}"/pythonpath_prepend.bash "@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@") + +# to avoid side effect with multiple workspace having different version of catkin_pip +unset CATKIN_PIP_SCRIPTS diff --git a/cmake/nosetests.cmake.in b/cmake/nosetests.cmake.in index ea9bbbf..6c3fcca 100644 --- a/cmake/nosetests.cmake.in +++ b/cmake/nosetests.cmake.in @@ -92,7 +92,7 @@ find_program(PIP_NOSETESTS NAMES "nosetests-${PYTHON_VERSION_MAJOR}" "nosetests" PATHS - ${CATKIN_DEVEL_PREFIX}/@CATKIN_GLOBAL_BIN_DESTINATION@ + ${CATKIN_PIP_ENV}/@CATKIN_GLOBAL_BIN_DESTINATION@ NO_DEFAULT_PATH ) if(PIP_NOSETESTS) diff --git a/cmake/pytest.cmake.in b/cmake/pytest.cmake.in index a3249ab..2453d40 100644 --- a/cmake/pytest.cmake.in +++ b/cmake/pytest.cmake.in @@ -92,7 +92,7 @@ find_program(PIP_PYTEST NAMES "py.test-${PYTHON_VERSION_MAJOR}" "py.test" PATHS - ${CATKIN_DEVEL_PREFIX}/@CATKIN_GLOBAL_BIN_DESTINATION@ + ${CATKIN_PIP_ENV}/@CATKIN_GLOBAL_BIN_DESTINATION@ NO_DEFAULT_PATH ) if(PIP_PYTEST) diff --git a/cmake/templates/python_setuptools_install.bat.in b/cmake/templates/python_setuptools_install.bat.in deleted file mode 100644 index 4c55da6..0000000 --- a/cmake/templates/python_setuptools_install.bat.in +++ /dev/null @@ -1,17 +0,0 @@ -@echo off - -if DEFINED DESTDIR ( - echo "Destdir.............%DESTDIR%" - set DESTDIR_ARG="--single-version-externally-managed --root=%DESTDIR%" -) - -cd "@INSTALL_CMD_WORKING_DIRECTORY@" - -cmd /V:on /C set PYTHONPATH="@CMAKE_INSTALL_PREFIX@/@PYTHON_INSTALL_DIR@;@CMAKE_BINARY_DIR@/@PYTHON_INSTALL_DIR@" ^ - && set CATKIN_BINARY_DIR="@CMAKE_BINARY_DIR@" ^ - && "@PYTHON_EXECUTABLE@" ^ - "@CATKIN_PIP_PACKAGE_PATH@/setup.py" ^ - build --build-base "@CMAKE_CURRENT_BINARY_DIR@" ^ - install ^ - %DESTDIR_ARG% ^ - @SETUPTOOLS_ARG_EXTRA@ --prefix="@CMAKE_INSTALL_PREFIX@" --install-scripts="@CMAKE_INSTALL_PREFIX@/@CATKIN_GLOBAL_BIN_DESTINATION@" diff --git a/cmake/templates/python_setuptools_install.sh.in b/cmake/templates/python_setuptools_install.sh.in index 478440a..ada3274 100755 --- a/cmake/templates/python_setuptools_install.sh.in +++ b/cmake/templates/python_setuptools_install.sh.in @@ -9,25 +9,33 @@ if [ -n "$DESTDIR" ] ; then /bin/echo "otherwise python's distutils will bork things." exit 1 esac - DESTDIR_ARG="--single-version-externally-managed --root=$DESTDIR" + DESTDIR_ARG="--root=$DESTDIR" +else + DESTDIR_ARG="--record=@CMAKE_INSTALL_PREFIX@/@PROJECT_NAME@.record" fi echo_and_run() { echo "+ $@" ; "$@" ; } echo_and_run cd "@INSTALL_CMD_WORKING_DIRECTORY@" -# snsure that Python install destination exists -echo_and_run mkdir -p "$DESTDIR@CMAKE_INSTALL_PREFIX@/@PYTHON_INSTALL_DIR@" +# Ensure that Python install destination exists +# Not needed with latest setuptools ? +#echo_and_run mkdir -p "$DESTDIR@CMAKE_INSTALL_PREFIX@/@PYTHON_INSTALL_DIR@" + +# Verifying setuptools version +echo_and_run /usr/bin/env \ + PYTHONPATH="@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@:@CMAKE_INSTALL_PREFIX@/@PYTHON_INSTALL_DIR@:@CMAKE_BINARY_DIR@/@PYTHON_INSTALL_DIR@:$PYTHONPATH" \ + python -c "import setuptools; print 'setuptools from {0} version {1}'.format(setuptools.__file__, setuptools.__version__)" # Note that PYTHONPATH is pulled from the environment to support installing # into one location when some dependencies were installed in another # location, #123. +# One of these locations must be the catkin_pip_env with the latest setuptools version to install packages with recent setuptools. echo_and_run /usr/bin/env \ - PYTHONPATH="@CMAKE_INSTALL_PREFIX@/@PYTHON_INSTALL_DIR@:@CMAKE_BINARY_DIR@/@PYTHON_INSTALL_DIR@:$PYTHONPATH" \ - CATKIN_BINARY_DIR="@CMAKE_BINARY_DIR@" \ + PYTHONPATH="@CATKIN_PIP_ENV@/@CATKIN_PIP_PYTHON_INSTALL_DIR@:@CMAKE_INSTALL_PREFIX@/@PYTHON_INSTALL_DIR@:@CMAKE_BINARY_DIR@/@PYTHON_INSTALL_DIR@:$PYTHONPATH" \ "@PYTHON_EXECUTABLE@" \ "@CATKIN_PIP_PACKAGE_PATH@/setup.py" \ build --build-base "@CMAKE_CURRENT_BINARY_DIR@" \ - install \ + install --single-version-externally-managed \ $DESTDIR_ARG \ @SETUPTOOLS_ARG_EXTRA@ --prefix="@CMAKE_INSTALL_PREFIX@" --install-scripts="@CMAKE_INSTALL_PREFIX@/@CATKIN_GLOBAL_BIN_DESTINATION@" From 29b1e3d2a6d5b84539aa8e40e3a080ebeffc7972 Mon Sep 17 00:00:00 2001 From: alexv Date: Tue, 9 Aug 2016 15:12:36 +0900 Subject: [PATCH 2/2] making sure env-hooks have all variables setup before adding. --- cmake/catkin-pip.cmake.in | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmake/catkin-pip.cmake.in b/cmake/catkin-pip.cmake.in index 0f0deb0..56e0a05 100644 --- a/cmake/catkin-pip.cmake.in +++ b/cmake/catkin-pip.cmake.in @@ -26,14 +26,15 @@ if ( NOT CATKIN_PIP_ENV_HOOKS_PATH ) set (CATKIN_PIP_ENV_HOOKS_PATH ${CMAKE_CURRENT_LIST_DIR}/env-hooks) endif() -# Since we need the envhook as soon as a package is using catkin-pip from source, but only in devel space -catkin_add_env_hooks(42.site_packages SHELLS bash DIRECTORY ${CATKIN_PIP_ENV_HOOKS_PATH}) -# TODO : cmake should include this file only once, so the env hook is not recreated for every package that includes it... - # We want the catkin environment to be used for both devel and install workspace # to use latest versions python tools in both. set(CATKIN_PIP_ENV ${CMAKE_BINARY_DIR}/catkin_pip_env CACHE PATH "The path containing the python environment for catkin_pip dependencies" ) set(CATKIN_PIP_PYTHON_INSTALL_DIR @CATKIN_PIP_GLOBAL_PYTHON_DESTINATION@) + +# Since we need the envhook as soon as a package is using catkin-pip from source, but only in devel space +catkin_add_env_hooks(42.site_packages SHELLS bash DIRECTORY ${CATKIN_PIP_ENV_HOOKS_PATH}) +# TODO : cmake should include this file only once, so the env hook is not recreated for every package that includes it... + catkin_pip_setup_prefix(${CATKIN_PIP_ENV}) #