Restructure general workflow, cli, services, processors, docs, tests

.gitignore

@@ -107,3 +107,6 @@ env2/
 ocrd.egg-info
 /src
 spec
+.pytest_cache
+.~lock*
+/profile

.pylintrc

@@ -0,0 +1,13 @@
+[MASTER]
+extension-pkg-whitelist=lxml
+ignored-modules=cv2,tesserocr
+
+[MESSAGES CONTROL]
+disable =
+    missing-docstring,
+    no-self-use,
+    too-many-arguments,
+    superfluous-parens,
+    invalid-name,
+    line-too-long,
+    too-few-public-methods,

.travis.yml

@@ -0,0 +1,30 @@
+language: python
+python:
+  - 2.7
+  - 3.6
+before_install:
+  - sudo apt-get -qq update
+  - sudo apt-get install -y autoconf automake libtool
+  - sudo apt-get install -y libpng12-dev
+  - sudo apt-get install -y libjpeg62-dev
+  - sudo apt-get install -y libtiff4-dev
+  - sudo apt-get install -y zlib1g-dev
+  - wget http://www.leptonica.org/source/leptonica-1.73.tar.gz -O /tmp/leptonica.tar.gz
+  - tar -xvf /tmp/leptonica.tar.gz
+  - pushd leptonica-1.73 && ./configure && make && sudo make install && popd
+  - wget https://github.com/tesseract-ocr/tesseract/archive/3.04.01.tar.gz -O /tmp/tesseract.tar.gz
+  - tar -xvf /tmp/tesseract.tar.gz
+  - cd tesseract-3.04.01
+  - ./autogen.sh && ./configure
+  - LDFLAGS="-L/usr/local/lib" CFLAGS="-I/usr/local/include" make
+  - sudo make install && sudo ldconfig
+  - cd ..
+  - wget https://github.com/tesseract-ocr/tessdata/archive/3.04.00.tar.gz -O /tmp/tessdata.tar.gz
+  - tar -xvf /tmp/tessdata.tar.gz
+  - sudo mkdir -p /usr/local/share/tessdata/
+  - sudo rsync -a tessdata-3.04.00/ /usr/local/share/tessdata
+  - sudo apt-get install -y libimage-exiftool-perl libxml2-utils
+install:
+  - make deps-pip test-deps-pip
+script:
+  - make test

Makefile

@@ -1,14 +1,25 @@
+export
+
+SHELL = /bin/bash
+PYTHON = python2
+PYTHONPATH := .:$(PYTHONPATH)
+PIP = pip
+LOG_LEVEL = INFO
+
 # BEGIN-EVAL makefile-parser --make-help Makefile
 
 help:
 	@echo ""
 	@echo "  Targets"
 	@echo ""
-	@echo "    deps-ubuntu  Dependencies for deployment in an ubuntu/debian linux"
-	@echo "    deps-pip     Install python deps via pip"
-	@echo "    spec         Clone the spec dir for sample files"
-	@echo "    install      (Re)install the tool"
-	@echo "    test-run     Test the run command"
+	@echo "    deps-ubuntu    Dependencies for deployment in an ubuntu/debian linux"
+	@echo "    deps-pip       Install python deps via pip"
+	@echo "    spec           Clone the spec dir for sample files"
+	@echo "    install        (Re)install the tool"
+	@echo "    test-deps-pip  Install test python deps via pip"
+	@echo "    test           Run all unit tests"
+	@echo "    docs           Build documentation"
+	@echo "    docs-clean     Clean docs"
 
 # END-EVAL
 
@@ -20,22 +31,51 @@ deps-ubuntu:
 		libtesseract-dev \
 		libleptonica-dev \
 		libimage-exiftool-perl \
+		libxml2-utils \
 		tesseract-ocr-eng \
 		tesseract-ocr-deu \
 		tesseract-ocr-deu-frak
 
 # Install python deps via pip
 deps-pip:
-	pip3 install --user -r requirements.txt
+	$(PIP) install -r requirements.txt
 
 # Clone the spec dir for sample files
 spec:
 	git clone https://github.com/OCR-D/spec
 
 # (Re)install the tool
 install:
-	pip3 install --user .
+	$(PIP) install .
+
+test/assets: spec
+	mkdir -p test/assets
+	cp -r spec/io/example test/assets/herold
+
+# Install test python deps via pip
+test-deps-pip:
+	$(PIP) install -r requirements.txt
+
+.PHONY: test
+# Run all unit tests
+test:
+	$(PYTHON) -m pytest --log-level=$(LOG_LEVEL) --duration=10 test
+
+.PHONY: docs
+# Build documentation
+docs:
+	sphinx-apidoc -f -o docs/api ocrd
+	cd docs ; $(MAKE) html
+
+# Clean docs
+docs-clean:
+	cd docs ; rm -rf _build api
+
+pyclean:
+	rm **/*.pyc
+	rm -rf .pytest_cache
+
+test-profile:
+	$(PYTHON) -m cProfile -o profile $(which py.test) test
+	$(PYTHON) analyze_profile.py
 
-# Test the run command
-test-run: spec
-	run-ocrd spec/io/example/mets.xml

README.rst

@@ -10,22 +10,25 @@ To bootstrap the tool, you'll need installed (Ubuntu packages):
 
 * Python (``python``)
 * pip (``python-pip``)
-* Tesseract (3.04) headers (``libtesseract-dev``)
-* Some tesseract (3.04) language models (``tesseract-ocr-{eng,deu,deu-frak,...}``)
+* Tesseract headers (``libtesseract-dev``)
+* Some tesseract language models (``tesseract-ocr-{eng,deu,deu-frak,...}``)
 * Leptonica headers (``libleptonica-dev``)
 * exiftool (``libimage-exiftool-perl``)
+* libxml2-utils for xmllint (``libxml2-utils``)
 
 To install system-wide:
 
 ::
 
+    sudo make deps-ubuntu
     pip install -r requirements.txt
     pip install .
 
 To install to user HOME dir
 
 ::
 
+    sudo make deps-ubuntu
     pip install --user -r requirements.txt
     pip install .
 
@@ -48,21 +51,38 @@ If tesserocr fails to compile with an error:::
 
 This is due to some inconsistencies in the installed tesseract C headers. Replace ``string`` with ``std::string`` in ``$PREFIX/include/tesseract/unicharset.h:265:5:`` and ``$PREFIX/include/tesseract/unichar.h:164:10:`` ff.
 
-If tesserocr fails with an error about ``LSTM``/``CUBE``, you are using th 4.00
-headers. Downgrade to 3.04: ``apt install libtesseract-dev=3.04.01-6`` or
-whatever ``apt policy libtesseract-dev`` offers. Make sure there are no spurious pkg-config artifacts, e.g. in ``/usr/local/lib/pkgconfig/tesseract.pc``. The same goes for language models
+If tesserocr fails with an error about ``LSTM``/``CUBE``, you are have a
+mismatch between tesseract header/data/pkg-config versions. ``apt policy
+libtesseract-dev`` lists the apt-installable versions, keep it consistent. Make
+sure there are no spurious pkg-config artifacts, e.g. in
+``/usr/local/lib/pkgconfig/tesseract.pc``. The same goes for language models.
 
 
 Usage
 -----
 
+pyocrd installs a binary ``ocrd`` that can be used to invoke the processors
+directly (``ocrd process``) or start (development) webservices (``ocrd server``)
+
+Examples:
+
 ::
 
-    run-ocrd <METS-FILE>
+    # List available processors
+    ocrd process
+
+    # Region-segment with tesserocr all files in METS INPUT fileGrp
+    ocrd process -m /path/to/mets.xml segment-region/tesserocr
+
+    # Chain multiple processors
+    ocrd process -m /path/to/mets.xml characterize/exif segment-line/tesserocr recognize/tesserocr
 
-This will run the image characterization, page segmentation and region segmentation.
+    # Start a processor web service at port 6543
+    ocrd server process -p 6543
+    http PUT localhost:6543/characterize url==http://server/path/to/mets.xml
 
 See Also
 --------
 
 * `OCR-D Specifications <https://github.com/ocr-d/spec>`_
+* `pyocrd wiki <https://github.com/ocr-d/pyocrd/wiki>`_

analyze_profile.py

@@ -0,0 +1,5 @@
+import pstats
+p = pstats.Stats('profile')
+p.strip_dirs()
+p.sort_stats('tottime')
+p.print_stats(50)

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = pyocrd
+SOURCEDIR     = .
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/api/modules.rst

@@ -0,0 +1,7 @@
+ocrd
+====
+
+.. toctree::
+   :maxdepth: 4
+
+   ocrd

docs/api/ocrd.cli.rst

@@ -0,0 +1,30 @@
+ocrd.cli package
+================
+
+Submodules
+----------
+
+ocrd.cli.merge\_ocr\_txt module
+-------------------------------
+
+.. automodule:: ocrd.cli.merge_ocr_txt
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+ocrd.cli.run module
+-------------------
+
+.. automodule:: ocrd.cli.run
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: ocrd.cli
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/api/ocrd.model.rst

@@ -0,0 +1,46 @@
+ocrd.model package
+==================
+
+Submodules
+----------
+
+ocrd.model.ocrd\_file module
+----------------------------
+
+.. automodule:: ocrd.model.ocrd_file
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+ocrd.model.ocrd\_mets module
+----------------------------
+
+.. automodule:: ocrd.model.ocrd_mets
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+ocrd.model.ocrd\_page module
+----------------------------
+
+.. automodule:: ocrd.model.ocrd_page
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+ocrd.model.ocrd\_xml\_base module
+---------------------------------
+
+.. automodule:: ocrd.model.ocrd_xml_base
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: ocrd.model
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/api/ocrd.processor.characterize.rst

@@ -0,0 +1,22 @@
+ocrd.processor.characterize package
+===================================
+
+Submodules
+----------
+
+ocrd.processor.characterize.exif module
+---------------------------------------
+
+.. automodule:: ocrd.processor.characterize.exif
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: ocrd.processor.characterize
+    :members:
+    :undoc-members:
+    :show-inheritance:

docs/api/ocrd.processor.rst

@@ -0,0 +1,31 @@
+ocrd.processor package
+======================
+
+Subpackages
+-----------
+
+.. toctree::
+
+    ocrd.processor.characterize
+    ocrd.processor.segment_line
+    ocrd.processor.segment_region
+
+Submodules
+----------
+
+ocrd.processor.base module
+--------------------------
+
+.. automodule:: ocrd.processor.base
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: ocrd.processor
+    :members:
+    :undoc-members:
+    :show-inheritance: