Easyinstall fix

.gitignore

@@ -3,3 +3,4 @@ build
 testbuild
 test/mypippkg
 .idea
+*.pyc

.travis.yml

@@ -13,19 +13,19 @@ env:
   # This will check any ROS distro supported on this OS
   # checking devel and install separately so that they don't influence each other (dependencies, path, env, etc.)
   matrix:
-    - CI_ROS_DISTRO=indigo ROS_FLOW=devel
-    - CI_ROS_DISTRO=indigo ROS_FLOW=install
-    - CI_ROS_DISTRO=jade ROS_FLOW=devel
-    - CI_ROS_DISTRO=jade ROS_FLOW=install
-    - CI_ROS_DISTRO=kinetic ROS_FLOW=devel
-    - CI_ROS_DISTRO=kinetic ROS_FLOW=install
+    - ROS_DISTRO=indigo ROS_FLOW=devel
+    - ROS_DISTRO=indigo ROS_FLOW=install
+    - ROS_DISTRO=jade ROS_FLOW=devel
+    - ROS_DISTRO=jade ROS_FLOW=install
+    - ROS_DISTRO=kinetic ROS_FLOW=devel
+    - ROS_DISTRO=kinetic ROS_FLOW=install
   # TODO : test all possible flows (installing deps on devel or not, etc.)
 
 before_install:
   # Getting docker ros image
-  - docker pull ros:${CI_ROS_DISTRO}-ros-core
+  - docker pull ros:${ROS_DISTRO}-ros-core
   # Running as daemon
-  - docker run --name ${CONTAINER_NAME} -d -t ros:${CI_ROS_DISTRO}-ros-core /bin/bash | tee container.id
+  - docker run --name ${CONTAINER_NAME} -d -t ros:${ROS_DISTRO}-ros-core /bin/bash | tee container.id
   # Checking current container
   - docker ps -a
   - docker exec -ti ${CONTAINER_NAME} hostname
@@ -41,15 +41,15 @@ install:
   # copying local clone to the running container (volume is currently broken)
   - docker cp . ${CONTAINER_NAME}:/git_clone
   # Installing package dependencies
-  - docker exec -ti ${CONTAINER_NAME} rosdep install --default-yes --from-paths /git_clone --rosdistro $CI_ROS_DISTRO
+  - docker exec -ti ${CONTAINER_NAME} rosdep install --default-yes --from-paths /git_clone --rosdistro $ROS_DISTRO
 
 # full ROS setup, build and test
 script:
   - CONTAINER_ID=$(cat container.id)
   - docker ps -a
-  - docker exec -ti ${CONTAINER_NAME} /bin/bash -c "source /opt/ros/$CI_ROS_DISTRO/setup.bash && rospack profile"
+  - docker exec -ti ${CONTAINER_NAME} /bin/bash -c "source /opt/ros/$ROS_DISTRO/setup.bash && rospack profile"
   # Passing env vars here since passing in docker run currently breaks (2016-08-25)
-  - docker exec -ti ${CONTAINER_NAME} /bin/bash -c "export CI_ROS_DISTRO=$CI_ROS_DISTRO && export ROS_FLOW=$ROS_FLOW && /git_clone/travis_checks.bash"
+  - docker exec -ti ${CONTAINER_NAME} /bin/bash -c "export ROS_DISTRO=$ROS_DISTRO && export ROS_FLOW=$ROS_FLOW && /git_clone/travis_checks.bash"
   - docker stop "${CONTAINER_ID}"
 
 notifications:

CMakeLists.txt

@@ -32,7 +32,7 @@ catkin_package(
   CFG_EXTRAS
   catkin-pip.cmake
   catkin-pip-runcmd.cmake
-  catkin-pip-setup.cmake
+  catkin-pip-package.cmake
   catkin-pip-prefix.cmake
   catkin-pip-requirements.cmake
   pytest.cmake
@@ -45,7 +45,6 @@ configure_file(cmake/catkin-pip-fixups.req ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKA
 configure_file(cmake/scripts/path_prepend.sh ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/scripts/path_prepend.sh COPYONLY)
 configure_file(cmake/env-hooks/42.site_packages.sh.installspace.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks/42.site_packages.sh.installspace.in COPYONLY)
 configure_file(cmake/env-hooks/42.site_packages.sh.develspace.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks/42.site_packages.sh.develspace.in COPYONLY)
-configure_file(cmake/env-hooks/42.site_packages.bash.develspace.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks/42.site_packages.bash.develspace.in COPYONLY)
 #configure_file(cmake/templates/python_setuptools_install.bat.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/templates/python_setuptools_install.bat.in COPYONLY)
 configure_file(cmake/templates/python_setuptools_install.sh.in ${CATKIN_DEVEL_PREFIX}/${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/templates/python_setuptools_install.sh.in COPYONLY)
 
@@ -67,7 +66,6 @@ install(PROGRAMS
 install(FILES
     cmake/env-hooks/42.site_packages.sh.develspace.in
     cmake/env-hooks/42.site_packages.sh.installspace.in
-    cmake/env-hooks/42.site_packages.bash.develspace.in
     DESTINATION
     ${CATKIN_PACKAGE_SHARE_DESTINATION}/cmake/env-hooks
 )

cmake/catkin-pip-base.req

@@ -2,5 +2,6 @@
 # It is better here to pin packages to a verified version, especially since we are explointing a borderline usecase
 # However this means we need a short maintenance cycle to not be outdated by python folks...
 
-pip==8.1.2  # pip > 9.0 tries to remove egg-info/entry_points.txt from /opt/yujin/amd64/indigo-devel/lib/python2.7/dist-packages/pyros-... even if we specify /home/alexv/ROS/gopher_bootstrap/build/catkin_pip_env/bin/pip install -e /home/alexv/ROS/gopher_bootstrap/src/pyros --ignore-installed --no-dependencies --prefix /home/alexv/ROS/gopher_bootstrap/devel
-setuptools==32.3.1  # we hit a bug in 33.0.0 on undefined setuptools.Command...
+pip==8.1.2  # pip > 9.0 tries to remove system packages data, even if we specify /home/alexv/ROS/gopher_bootstrap/build/catkin_pip_env/bin/pip install -e /home/alexv/ROS/gopher_bootstrap/src/pyros --ignore-installed --no-dependencies --prefix /home/alexv/ROS/gopher_bootstrap/devel
+# Issue : https://github.com/asmodehn/catkin_pip/issues/58
+setuptools==33.1.1

cmake/catkin-pip-setup.cmake.in → cmake/catkin-pip-package.cmake.in

@@ -1,5 +1,3 @@
-message(STATUS "Loading catkin-pip-setup.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
-
 if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 	message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" )
 endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
@@ -10,6 +8,9 @@ if(catkin_pip_setup_included)
 endif(catkin_pip_setup_included)
 set(catkin_pip_setup_included true)
 
+message(STATUS "Loading catkin-pip-package.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
+
+
 # protecting against missing cmake file
 include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-runcmd.cmake" RESULT_VARIABLE CATKIN_PIP_RUNCMD_FOUND )
 IF ( NOT CATKIN_PIP_RUNCMD_FOUND )
@@ -75,7 +76,10 @@ function(catkin_pip_python_setup)
 
 endfunction()
 
-function(catkin_pip_setup)
+# TODO : we now create an actual cmake target -> rename to catkin_pip_target ?
+function(catkin_pip_package package_name)
+
+    set(${PROJECT_NAME}_PIP_TARGET ${package_name} CACHE STRING "Make target generated to install this pip package as --editable for development")
 
     set (extra_macro_args ${ARGN})
 
@@ -91,21 +95,29 @@ function(catkin_pip_setup)
     # Note : environment should already be setup at configure time for devel
 
     # Then we can run the pip command
+    # Note when installing in editable mode (for development) we shouldnt care about already installed versions.
+    # However : https://github.com/asmodehn/catkin_pip/issues/58
     if(CATKIN_PIP_NO_DEPS)
-        catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --no-dependencies --prefix "${CATKIN_DEVEL_PREFIX}")
+        catkin_pip_install_devel_target(${package_name} ${package_path} --no-dependencies --ignore-installed)
+        #catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --no-dependencies --prefix "${CATKIN_DEVEL_PREFIX}" --ignore-installed)
     else()
-        catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --prefix "${CATKIN_DEVEL_PREFIX}")
+        catkin_pip_install_devel_target(${package_name} ${package_path} --ignore-installed)
+        #catkin_pip_runcmd(${CATKIN_PIP} install -e ${package_path} --prefix "${CATKIN_DEVEL_PREFIX}" --ignore-installed)
     endif()
 
     if(NOT EXISTS ${package_path}/setup.py)
         message(FATAL_ERROR "catkin_pip_setup() called without 'setup.py' in project folder '${package_path}'")
     endif()
 
+    # Probably better to do that here (dont do anything "special" for python)
     catkin_pip_python_setup()
 
     # TODO : we might want to generate a package.xml on the fly from setup.py contents...
 
     # Hijacking catkin scripts again
     #set(_PACKAGE_XML_DIRECTORY ${package_path})
 
+    # Here we plug back into usual catkin package build flow
+    catkin_package()
+
 endfunction()

cmake/catkin-pip-prefix.cmake.in

@@ -1,20 +1,22 @@
-message(STATUS "Loading catkin-pip-prefix.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
-
 if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 	message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" )
 endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 
-#
-# Important : This script is included by multiple cmake scripts at configure and build time
-# So it needs to be idempotent (no change if called multiple time with same settings/environment)
-#
-
 # Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard
 if(catkin_pip_prefix_included)
   return()
 endif(catkin_pip_prefix_included)
 set(catkin_pip_prefix_included true)
 
+message(STATUS "Loading catkin-pip-prefix.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
+
+
+#
+# Important : This script is included by multiple cmake scripts at configure and build time
+# So it needs to be idempotent (no change if called multiple time with same settings/environment)
+#
+
+
 # protecting against missing cmake file
 include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-runcmd.cmake" RESULT_VARIABLE CATKIN_PIP_RUNCMD_FOUND )
 IF ( NOT CATKIN_PIP_RUNCMD_FOUND )
@@ -116,22 +118,19 @@ endfunction(find_catkin_system_pip)
 
 function(catkin_pip_setup_prefix ws_prefix)
 
-    # Setting up our environment (for devel space only)
-    #catkin_pip_check_env(${ws_prefix})
-    # TODO : make this an env hook and use it via CATKIN_ENV
-
     # Trying to find our own pip
     # Careful this creates a CACHE variable that we need to recreate here in case people clean devel without cleaning build
     unset(CATKIN_PIP CACHE)
     find_program(CATKIN_PIP NAMES pip pip2 pip2.7 PATHS ${ws_prefix}/@CATKIN_GLOBAL_BIN_DESTINATION@ NO_DEFAULT_PATH)
 
     if (CATKIN_PIP)
-        message(STATUS "    ... Catkin pip was found at ${CATKIN_PIP} ...")
         set(CATKIN_PIP "${CATKIN_PIP} -q")  # we can add all default basic options here.
     else ()
         # If not found, it means we need to do the whole setup...
         unset(CATKIN_PIP CACHE)
+        # TODO:  cleanup if unnecessary message (the find_program already outputs something
         message(STATUS "    ... Catkin pip was not found in ${ws_prefix}/@CATKIN_GLOBAL_BIN_DESTINATION@ ...")
+
         # Assuming Ubuntu Trusty here. platform detection is another hurdle
         set(CMAKE_SYSTEM_PREFIX_PATH / /usr /usr/local)
 

cmake/catkin-pip-requirements.cmake.in

@@ -1,5 +1,3 @@
-message(STATUS "Loading catkin-pip-requirements.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
-
 if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 	message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" )
 endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
@@ -10,6 +8,9 @@ if(catkin_pip_requirements_included)
 endif(catkin_pip_requirements_included)
 set(catkin_pip_requirements_included true)
 
+message(STATUS "Loading catkin-pip-requirements.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
+
+
 # protecting against missing cmake file
 include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-prefix.cmake" RESULT_VARIABLE CATKIN_PIP_PREFIX_FOUND )
 IF ( NOT CATKIN_PIP_PREFIX_FOUND )

cmake/catkin-pip-runcmd.cmake.in

@@ -1,25 +1,26 @@
-message(STATUS "Loading catkin-pip-runcmd.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
-
 if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 	message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" )
 endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 
-#
-# Important : This script is included by multiple cmake scripts at configure and build time
-# So it needs to be idempotent (no change if called multiple time with same settings/environment)
-#
-
 # Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard
 if(catkin_pip_runcmd_included)
   return()
 endif(catkin_pip_runcmd_included)
 set(catkin_pip_runcmd_included true)
 
+message(STATUS "Loading catkin-pip-runcmd.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
+
+#
+# Important : This script is included by multiple cmake scripts at configure and build time
+# So it needs to be idempotent (no change if called multiple time with same settings/environment)
+#
+
 
 function(catkin_pip_runcmd)
 
     set(CATKIN_PIP_COMMAND ${ARGN})
 
+    # TODO : refactor to use separate arguments, like catkin_pip_install_devel_target
     string(REPLACE ";" " " CATKIN_PIP_CMDSTR "${CATKIN_ENV} ${CATKIN_PIP_COMMAND}")
     message(STATUS "    ... Running ${CATKIN_PIP_CMDSTR} ...")
 
@@ -38,4 +39,47 @@ function(catkin_pip_runcmd)
         message(FATAL_ERROR "${PIP_ERROR}")
     endif()
 
-endfunction()
+endfunction()
+
+
+function(catkin_pip_install_devel_target package_name package_path)
+    set(CATKIN_PIP_INSTALL_DEVEL_OUTPUTS
+        # ${CATKIN_DEVEL_PREFIX}/${CATKIN_PIP_PYTHON_INSTALL_DIR}/easy-install.pth
+        # CAREFUL with multiple outputs : https://cmake.org/Bug/view.php?id=15116
+        ${CATKIN_DEVEL_PREFIX}/${CATKIN_PIP_PYTHON_INSTALL_DIR}/${package_name}.egg-link
+    )
+
+    string(REPLACE ";" " " ARGN_STR "${ARGN}")
+
+    # note setlock because of https://github.com/pypa/pip/issues/2361
+    # set(CATKIN_PIP_CMD ${CATKIN_ENV})
+    # set(CATKIN_PIP_CMD_ARGS_STR "setlock ${CATKIN_PIP_ENV}/catkin_pip.lock ${CATKIN_PIP} install -e ${package_path} --prefix ${CATKIN_DEVEL_PREFIX} ${ARGN_STR}")
+    # separate_arguments(CATKIN_PIP_CMD_ARGS_LIST UNIX_COMMAND ${CATKIN_PIP_CMD_ARGS_STR})
+    # TODO : change to use --target to allow installing duplicate packages with pip>=9.0.0 ... Ref https://github.com/pypa/pip/issues/4243
+
+    # TMP : since PYTHONPATH breaks pip install --editable Ref : https://github.com/pypa/pip/issues/4261
+    set(CATKIN_PIP_CMD /bin/bash)  # Ubuntu (Linux?) only
+    set(CATKIN_PIP_CMD_ARGS_STR "-c 'export PYTHONPATH=${CATKIN_DEVEL_PREFIX}/${CATKIN_PIP_PYTHON_INSTALL_DIR}\; setlock ${CATKIN_PIP_ENV}/catkin_pip.lock ${CATKIN_PIP} install -e ${package_path} --prefix ${CATKIN_DEVEL_PREFIX} ${ARGN_STR}'")
+    separate_arguments(CATKIN_PIP_CMD_ARGS_LIST UNIX_COMMAND ${CATKIN_PIP_CMD_ARGS_STR})
+
+    # Note that catkin_env is called first. That is because we do not trust catkin_env return code and async behavior.
+    # However we have to assume it doesn't alter the environment in a way that breaks setlock.
+    # Hint : on ubuntu trusty, I noticed the lock file stay there after execution, however the mutex behavior works fine.
+
+    add_custom_command(
+      OUTPUT ${CATKIN_PIP_INSTALL_DEVEL_OUTPUTS}
+
+      # Note we are using setlock to avoid concurrent pip calls (if make runs target in parallel)
+      # Note we override PYTHONPATH to workaround a pip bug
+      # Note we need to use catkin_env to make sure our envhook is properly loaded
+
+      COMMAND ${CATKIN_PIP_CMD} ${CATKIN_PIP_CMD_ARGS_LIST}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+      DEPENDS "${package_path}"  # file level dependency : package needs to be in the path
+      COMMENT "${CATKIN_PIP_CMD} ${CATKIN_PIP_CMD_ARGS_STR}"
+      VERBATIM
+    )
+
+    add_custom_target(${package_name} ALL DEPENDS ${CATKIN_PIP_INSTALL_DEVEL_OUTPUTS})  # file level dependency : we need the output of the custom command
+
+endfunction()

cmake/catkin-pip.cmake.in

@@ -1,9 +1,15 @@
-message(STATUS "Loading catkin-pip.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
-
 if ( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 	message ( FATAL_ERROR " CMAKE MINIMUM BACKWARD COMPATIBILITY REQUIRED : 2.8 !" )
 endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 
+# Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard
+if(catkin_pip_main_included)
+  return()
+endif(catkin_pip_main_included)
+set(catkin_pip_main_included true)
+
+message(STATUS "Loading catkin-pip.cmake from ${CMAKE_CURRENT_LIST_DIR}... ")
+
 # Setting required policies
 # required to be able to do "if (True)"
 # FOREACH(policy CMP0011 CMP0012 CMP0013 CMP0014)
@@ -12,23 +18,17 @@ endif( CMAKE_BACKWARDS_COMPATIBILITY LESS 2.8 )
 #     ENDIF()
 # ENDFOREACH()
 
-# Enforcing one time include https://cmake.org/Wiki/CMake_Performance_Tips#Use_an_include_guard
-if(catkin_pip_main_included)
-  return()
-endif(catkin_pip_main_included)
-set(catkin_pip_main_included true)
-
 # protecting against missing cmake file dependency
 include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-requirements.cmake" RESULT_VARIABLE CATKIN_PIP_REQUIREMENTS_FOUND )
 IF ( NOT CATKIN_PIP_REQUIREMENTS_FOUND )
     message ( FATAL_ERROR "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-requirements.cmake Not Found !!!" )
 ENDIF ( NOT CATKIN_PIP_REQUIREMENTS_FOUND )
 
 # protecting against missing cmake file dependency
-include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-setup.cmake" RESULT_VARIABLE CATKIN_PIP_SETUP_FOUND )
-IF ( NOT CATKIN_PIP_SETUP_FOUND )
-    message ( FATAL_ERROR "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-setup.cmake Not Found !!!" )
-ENDIF ( NOT CATKIN_PIP_SETUP_FOUND )
+include ( "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-package.cmake" RESULT_VARIABLE CATKIN_PIP_PACKAGE_FOUND )
+IF ( NOT CATKIN_PIP_PACKAGE_FOUND )
+    message ( FATAL_ERROR "${CMAKE_CURRENT_LIST_DIR}/catkin-pip-package.cmake Not Found !!!" )
+ENDIF ( NOT CATKIN_PIP_PACKAGE_FOUND )
 
 # Setting our paths to package env-hooks provided by catkin-pip
 if ( NOT CATKIN_PIP_ENV_HOOKS_PATH )
@@ -56,10 +56,7 @@ file(MAKE_DIRECTORY ${CATKIN_DEVEL_PREFIX}/@CATKIN_PIP_GLOBAL_PYTHON_DESTINATION
 
 # Since we need the envhook as soon as a package is using catkin-pip from source.
 catkin_add_env_hooks(42.site_packages SHELLS sh DIRECTORY ${CATKIN_PIP_ENV_HOOKS_PATH})
-# TODO : Also add envhooks for other shells
-catkin_add_env_hooks(42.site_packages SHELLS bash DIRECTORY ${CATKIN_PIP_ENV_HOOKS_PATH} SKIP_INSTALL)
-# SKIP_INSTALL because we definitely do not want it ending up in all deb packages built with catkin_pip.
-# If we need paths in install space, they should somehow be stored in generated scripts (make or others)
+# If we need paths in install space, they should somehow be stored in generated scripts (make or others) as much as possible
 # TODO : cmake should include this file only once per workspace, so the env hook is not recreated for every package that uses catkin_pip if they are in same workspace...