Merge pull request #198 from cmla/python_packaging

[refactor] make s2p a python package that can be installed with pip

.travis.yml

@@ -25,14 +25,13 @@ before_install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew unlink gdal && (brew install gcc || brew link --overwrite gcc) && brew install geographiclib fftw; fi
 
 install:
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install numpy utm bs4 requests pyproj; fi
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install rasterio; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install numpy; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install .; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then wget http://www.kyngchaos.com/files/software/frameworks/GDAL_Complete-2.1.dmg; fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then X=$(echo `hdiutil mount GDAL_Complete-2.1.dmg | tail -1 | awk '{$1=$2=""; print $0}'` | xargs -0 echo) && sudo installer -pkg "${X}/"GDAL\ Complete.pkg -target /; export PATH="/Library/Frameworks/GDAL.framework/Programs:$PATH"; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip2 install numpy utm bs4 requests pyproj; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip2 install rasterio[test]; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip2 install numpy; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then pip2 install .; fi
 
 script:
-  - make
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then python -u s2p_test.py --all; fi
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python2 -u s2p_test.py --all; fi
+  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then python -u tests/test_s2p.py --all; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then python2 -u tests/test_s2p.py --all; fi

MANIFEST.in

@@ -0,0 +1,113 @@
+include README.md
+include makefile
+include c/*.c
+include c/*.h
+include c/*.cpp
+include c/egm96-15.pgm
+include c/homography/Makefile
+include c/homography/linalg.c
+include c/homography/main.cpp
+include c/homography/pickopt.c
+include c/homography/LibHomography/*.cpp
+include c/homography/LibHomography/*.h
+include c/homography/LibImages/*.cpp
+include c/homography/LibImages/*.h
+include c/homography/Utilities/*.cpp
+include c/homography/Utilities/*.h
+include c/sift/Makefile
+include c/sift/linalg.c
+include c/sift/matching.c
+include c/sift/pickopt.c
+include c/sift/timing.c
+include c/sift/sift_roi.cpp
+include c/sift/sift_anatomy_20141201/*
+include c/sift/Utilities/*.cpp
+include c/sift/Utilities/*.h
+include c/sift/LibSift/*.cpp
+include c/sift/LibSift/*.h
+include c/sift/LibSSE/*.cpp
+include c/sift/LibSSE/*.h
+include c/sift/LibImages/*.cpp
+include c/sift/LibImages/*.h
+include 3rdparty/iio/iio.c
+include 3rdparty/iio/iio.h
+include 3rdparty/mgm_multi/refine.h
+include 3rdparty/mgm_multi/abstract_dsf.c
+include 3rdparty/mgm_multi/mgm_core.cc
+include 3rdparty/mgm_multi/main_mgm.cc
+include 3rdparty/mgm_multi/shear.h
+include 3rdparty/mgm_multi/dvec.cc
+include 3rdparty/mgm_multi/CMakeLists.txt
+include 3rdparty/mgm_multi/mgm_costvolume.cc
+include 3rdparty/mgm_multi/LICENSE
+include 3rdparty/mgm_multi/main_mgm_multi.cc
+include 3rdparty/mgm_multi/point.h
+include 3rdparty/mgm_multi/Makefile
+include 3rdparty/mgm_multi/stereo_utils.h
+include 3rdparty/mgm_multi/mgm_print_energy.h
+include 3rdparty/mgm_multi/point.cc
+include 3rdparty/mgm_multi/img.cc
+include 3rdparty/mgm_multi/stereo_utils.cc
+include 3rdparty/mgm_multi/smartparameter.h
+include 3rdparty/mgm_multi/img.h
+include 3rdparty/mgm_multi/abstract_dsf.h
+include 3rdparty/mgm_multi/mgm_multiscale.cc
+include 3rdparty/mgm_multi/shear.c
+include 3rdparty/mgm_multi/mgm_costvolume.h
+include 3rdparty/mgm_multi/mgm_refine.h
+include 3rdparty/mgm_multi/mgm_core.h
+include 3rdparty/mgm_multi/mgm_weights.h
+include 3rdparty/mgm_multi/mgm_multiscale.h
+include 3rdparty/mgm_multi/img_interp.h
+include 3rdparty/mgm_multi/census_tools.cc
+include 3rdparty/mgm_multi/remove_small_cc.c
+include 3rdparty/mgm_multi/data
+include 3rdparty/mgm_multi/iio
+include 3rdparty/mgm_multi/iio/CMakeLists.txt
+include 3rdparty/mgm_multi/iio/iio_test_named.c
+include 3rdparty/mgm_multi/iio/iio.c
+include 3rdparty/mgm_multi/iio/iioConfig.cmake.in
+include 3rdparty/mgm_multi/iio/FindOPENEXR.cmake
+include 3rdparty/mgm_multi/iio/iio.h
+include 3rdparty/mgm_multi/img_tools.h
+include 3rdparty/mgm/refine.h
+include 3rdparty/mgm/dvec2.cc
+include 3rdparty/mgm/mgm_core.cc
+include 3rdparty/mgm/dvec.cc
+include 3rdparty/mgm/point.h
+include 3rdparty/mgm/Makefile
+include 3rdparty/mgm/mgm_print_energy.h
+include 3rdparty/mgm/point.cc
+include 3rdparty/mgm/img.cc
+include 3rdparty/mgm/mgm.cc
+include 3rdparty/mgm/smartparameter.h
+include 3rdparty/mgm/img.h
+include 3rdparty/mgm/mgm_costvolume.h
+include 3rdparty/mgm/mgm_refine.h
+include 3rdparty/mgm/mgm_weights.h
+include 3rdparty/mgm/census_tools.cc
+include 3rdparty/mgm/iio/CMakeLists.txt
+include 3rdparty/mgm/iio/iio_test_named.c
+include 3rdparty/mgm/iio/iio.c
+include 3rdparty/mgm/iio/iioConfig.cmake.in
+include 3rdparty/mgm/iio/FindOPENEXR.cmake
+include 3rdparty/mgm/iio/iio.h
+include 3rdparty/mgm/img_tools.h
+include 3rdparty/tvl1flow/config.mk
+include 3rdparty/tvl1flow/bicubic_interpolation.c
+include 3rdparty/tvl1flow/zoom.c
+include 3rdparty/tvl1flow/Makefile
+include 3rdparty/tvl1flow/backflow.c
+include 3rdparty/tvl1flow/iio.c
+include 3rdparty/tvl1flow/main.c
+include 3rdparty/tvl1flow/mask.c
+include 3rdparty/tvl1flow/callTVL1.sh
+include 3rdparty/tvl1flow/xmalloc.c
+include 3rdparty/tvl1flow/README.txt
+include 3rdparty/tvl1flow/iio.h
+include 3rdparty/tvl1flow/tvl1flow_lib.c
+include 3rdparty/lsd/Makefile
+include 3rdparty/lsd/lsd.c
+include 3rdparty/lsd/lsd_cmd.c
+include 3rdparty/lsd/lsd.h
+include 3rdparty/lsd/lsd_call_example.c

README.md

@@ -2,11 +2,11 @@
 
 [![Build Status](https://travis-ci.org/MISS3D/s2p.svg?branch=master)](https://travis-ci.org/MISS3D/s2p)
 
-This software implements a stereo pipeline which produces elevation models from
-images taken by high resolution optical satellites such as Pléiades, WorldView,
-QuickBird, Spot or Ikonos. It generates 3D point clouds and digital surface
-models from stereo pairs (two images) or tri-stereo sets (three images) in a
-completely automatic fashion.
+S2P is a Python library and command line tool that implements a stereo
+pipeline which produces elevation models from images taken by high resolution
+optical satellites such as Pléiades, WorldView, QuickBird, Spot or Ikonos. It
+generates 3D point clouds and digital surface models from stereo pairs (two
+images) or tri-stereo sets (three images) in a completely automatic fashion.
 
 S2P was used to win the 2016 [IARPA Multi-View Stereo 3D Mapping Challenge](https://www.iarpa.gov/challenges/3dchallenge.html).
 
@@ -16,34 +16,97 @@ flavors of semi-global matching (SGM), TV-L1 optical flow, etc.
 The main language is Python, although several operations are handled by
 binaries written in C.
 
-The pipeline is implemented in the file `s2p.py`. The `s2p` module can be used
+The pipeline is implemented in the Python package `s2p`. It can be used
 to produce surface models and 3D point clouds from arbitrarily large regions
 of interest or from complete images. If needed, it cuts the region of interest
 in several small tiles and process them in parallel.
 
+Its main source code repository is https://github.com/miss3d/s2p.
+
+
+# Installation and dependencies
+
+Required dependencies (Ubuntu 16.04):
+
+    add-apt-repository -y ppa:ubuntugis/ppa  # The repository is added so that the version >= 2.1 of gdal is installed (requirement)
+    apt-get update
+    apt-get install build-essential cmake gdal-bin geographiclib-tools libgeographic-dev libfftw3-dev libgdal-dev libgeotiff-dev libtiff5-dev python python-numpy python-pip
+
+`gdal` version must be 2.1.0 or newer.
+
+## For users: install s2p from PyPI
+
+    pip install s2p
+
+## For developers: install s2p from sources
+
+    git clone https://github.com/MISS3D/s2p.git --recursive
+    cd s2p
+    pip install -e .
+
+The `--recursive` option for `git clone` allows to clone all git submodules, such
+as the [iio](https://github.com/mnhrdt/iio) library.
+
+If the `--recursive` option wasn't used when cloning, the submodules can now be
+retrieved with
+
+    git submodule update --init
+
+All `s2p` python submodules are located in the `s2p` package. Some python
+functions of these modules rely on external binaries. Most of these binaries
+were written on purpose for the needs of the pipeline, and their source code is
+provided here in the `c` folder. For the other binaries, the source code is
+provided in the `3rdparty` folder.
+
+All the sources (ours and 3rdparties) are compiled from the same makefile. Just
+run `make all` from the `s2p` folder to compile them. This will create a `bin`
+directory containing all the needed binaries. This makefile is used when
+running `pip install .`
+
+You can test if S2P is correctly working using:
+
+    make test
 
 ## Docker image
-[![Docker Status](http://dockeri.co/image/carlodef/s2p)](https://hub.docker.com/r/carlodef/s2p/)
+[![Docker Status](http://dockeri.co/image/cmla/s2p)](https://hub.docker.com/r/cmla/s2p/)
 
 A precompiled docker image is available and ready to use:
 
-    docker pull carlodef/s2p
+    docker pull cmla/s2p
+
+
+# Usage
+
+`s2p` is a Python library that can be imported into other applications. It also
+comes with a Command Line Interface (CLI).
 
+## From the command line
 
-## Usage
+The `s2p` CLI has an extensive help that can be printed with the `-h` and `--help` switches.
 
-Run the python script `s2p.py` with a json configuration file as unique
-argument:
+    $ s2p -h
+    usage: s2p.py [-h] config.json
 
-    python s2p.py testdata/input_pair/config.json
+    S2P: Satellite Stereo Pipeline
+
+    positional arguments:
+      config.json           path to a json file containing the paths to input and
+                            output files and the algorithm parameters
+
+    optional arguments:
+      -h, --help            show this help message and exit
+
+To run the whole pipeline, call `s2p` with a json configuration file as unique argument:
+
+    s2p tests/data/input_pair/config.json
 
 All the parameters of the algorithm, paths to input and output data are stored
 in the json file. See the provided `test.json` file for an example, and the
-comments in the file `s2plib/config.py` for some explanations about the roles
+comments in the file `s2p/config.py` for some explanations about the roles
 of these parameters.
 
-Notice that each input image must be accompanied by an `xml` file containing rpc
-coefficients.
+Notice that each input image must have RPC coefficients, either in its GeoTIFF
+tags or in a companion `.xml` or `.txt` file.
 
 #### ROI definition
 
@@ -64,48 +127,6 @@ the example).
 In the json configuration files, input and output paths are relative to the json
 file location, not to the current working directory.
 
-## Dependencies
-
-Required dependencies (Ubuntu 16.04):
-
-    add-apt-repository -y ppa:ubuntugis/ppa  # The repository is added so that the version >= 2.1 of gdal is installed (requirement)
-    apt-get update
-    apt-get install build-essential cmake gdal-bin geographiclib-tools libgeographic-dev libfftw3-dev libgdal-dev libgeotiff-dev libtiff5-dev python python-gdal python-numpy python-pip
-
-and
-
-    pip install utm bs4 lxml requests rasterio
-
-`gdal` version must be 2.1.0 or newer.
-
-
-## Installation
-
-    git clone https://github.com/MISS3D/s2p.git --recursive
-    cd s2p
-    make all
-
-The `--recursive` option for `git clone` allows to clone all submodules, such
-as the [iio](https://github.com/mnhrdt/iio) library.
-
-If the `--recursive` option wasn't used when cloning, the submodules can now be
-retrieved with
-
-    git submodule update --init
-
-All the python modules are located in the `s2plib` folder. Some python
-functions of these modules rely on external binaries. Most of these binaries
-were written on purpose for the needs of the pipeline, and their source code is
-provided here in the `c` folder. For the other binaries, the source code is
-provided in the `3rdparty` folder.
-
-All the sources (ours and 3rdparties) are compiled from the same makefile. Just
-run `make all` from the `s2p` folder to compile them.  This will create a `bin`
-directory containing all the needed binaries.
-
-You can test if S2P is correctly working using:
-
-    make test
 
 ### MicMac (optional)
 

makefile

@@ -31,7 +31,7 @@ all: default msmw3 sgbm mgm_multi
 
 # test for the default configuration
 test: default
-	python -u s2p_test.py --all
+	python -u tests/test_s2p.py --all
 
 # make sure that the destination directory is built
 $(BINDIR):

s2p.py → s2p/__init__.py

@@ -33,17 +33,17 @@
 import rasterio
 
 
-from s2plib.config import cfg
-from s2plib import common
-from s2plib import parallel
-from s2plib import initialization
-from s2plib import pointing_accuracy
-from s2plib import rectification
-from s2plib import block_matching
-from s2plib import masking
-from s2plib import triangulation
-from s2plib import fusion
-from s2plib import visualisation
+from s2p.config import cfg
+from s2p import common
+from s2p import parallel
+from s2p import initialization
+from s2p import pointing_accuracy
+from s2p import rectification
+from s2p import block_matching
+from s2p import masking
+from s2p import triangulation
+from s2p import fusion
+from s2p import visualisation
 
 
 def pointing_correction(tile, i):
@@ -781,21 +781,3 @@ def read_config_file(config_file):
                 img[d] = make_path_relative_to_file(img[d], config_file)
 
     return user_cfg
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description=('S2P: Satellite Stereo '
-                                                  'Pipeline'))
-    parser.add_argument('config', metavar='config.json',
-                        help=('path to a json file containing the paths to '
-                              'input and output files and the algorithm '
-                              'parameters'))
-    args = parser.parse_args()
-
-    user_cfg = read_config_file(args.config)
-
-    main(user_cfg)
-
-    # Backup input file for sanity check
-    if not args.config.startswith(os.path.abspath(cfg['out_dir']+os.sep)):
-        shutil.copy2(args.config,os.path.join(cfg['out_dir'],'config.json.orig'))

s2plib/block_matching.py → s2p/block_matching.py

@@ -6,8 +6,8 @@
 import os
 import numpy as np
 
-from s2plib import common
-from s2plib.config import cfg
+from s2p import common
+from s2p.config import cfg
 
 def rectify_secondary_tile_only(algo):
     if algo in ['tvl1_2d']: