diff --git a/.env b/.env index ed6c4072..f1671701 100644 --- a/.env +++ b/.env @@ -5,34 +5,43 @@ CONTROLLER_HOST=ocrd-controller CONTROLLER_ENV_UID=1001 CONTROLLER_ENV_GID=1001 CONTROLLER_ENV_UMASK=0002 -CONTROLLER_PORT_SSH=8022 -CONTROLLER_KEYS=./ocrd/controller/.ssh/authorized_keys -CONTROLLER_DATA=./kitodo/config_modules/metadata -CONTROLLER_MODELS=./ocrd/controller/models -CONTROLLER_CONFIG=./ocrd/controller/config -CONTROLLER_WORKERS=1 - -MANAGER_IMAGE=markusweigelt/ocrd_manager -MANAGER_IMAGE_TAG=latest +CONTROLLER_PORT_SSH=8022 +CONTROLLER_KEYS=${PWD}/ocrd/controller/.ssh/authorized_keys +CONTROLLER_DATA=${PWD}/kitodo/config_modules/metadata +CONTROLLER_MODELS=${PWD}/ocrd/controller/models +CONTROLLER_CONFIG=${PWD}/ocrd/controller/config +CONTROLLER_WORKERS=1 + +MONITOR_IMAGE=bertsky/ocrd_monitor +MONITOR_HOST=ocrd-monitor +MONITOR_PORT_WEB=5000 +MONITOR_PORT_GTK=8085 +MONITOR_DATA=${PWD}/kitodo/config_modules/metadata + +MANAGER_IMAGE=markusweigelt/ocrd_manager +MANAGER_IMAGE_TAG=latest MANAGER_HOST=ocrd-manager MANAGER_ENV_UID=1001 MANAGER_ENV_GID=1001 MANAGER_ENV_UMASK=0002 MANAGER_PORT_SSH=9022 +MANAGER_KEYS=${PWD}/ocrd/manager/.ssh/authorized_keys +MANAGER_KEY=${PWD}/ocrd/manager/.ssh/id_rsa +MANAGER_DATA=${PWD}/kitodo/config_modules/metadata -# Builder Kitodo.Production +# Builder Kitodo.Production BUILDER_TYPE=GIT BUILDER_GIT_COMMIT=ocrd-main BUILDER_GIT_SOURCE_URL=https://github.com/markusweigelt/kitodo-production/ -# Application Kitodo.Production -APP_BUILD_CONTEXT=./_modules/kitodo-production-docker/kitodo +# Application Kitodo.Production +APP_BUILD_CONTEXT=${PWD}/_modules/kitodo-production-docker/kitodo APP_BUILD_RESOURCES=build-resources APP_PORT=8080 APP_DEBUG=false APP_DEBUG_PORT=5005 -# Elastic Search +# Elastic Search ES_VERSION=7.17.3 ES_REST_PORT=9200 ES_NODE_PORT=9300 diff --git a/.env.stable b/.env.stable index e13cc4ba..48baaf79 100644 --- a/.env.stable +++ b/.env.stable @@ -7,9 +7,9 @@ CONTROLLER_ENV_UID=1001 CONTROLLER_ENV_GID=1001 CONTROLLER_ENV_UMASK=0002 CONTROLLER_PORT_SSH=8022 - -MANAGER_IMAGE=markusweigelt/ocrd_manager -MANAGER_IMAGE_TAG=ocrd-stable-20220405 + +MANAGER_IMAGE=markusweigelt/ocrd_manager +MANAGER_IMAGE_TAG=ocrd-stable-20220405 MANAGER_HOST=ocrd-manager MANAGER_ENV_UID=1001 MANAGER_ENV_GID=1001 diff --git a/Makefile b/Makefile index 41a8d49e..a3e76db1 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,18 @@ CONTROLLER_ENV_GID ?= $(shell id -g) MANAGER_ENV_UID ?= $(shell id -u) MANAGER_ENV_GID ?= $(shell id -g) +MODE ?= standalone +COMPOSE_FILES = docker-compose.yml +ifeq (managed,$(MODE)) +COMPOSE_FILES += docker-compose.managed.yml +endif +COMPOSE_FILES += docker-compose.kitodo-app.yml +COMPOSE_FILES += docker-compose.kitodo-app.override.yml +NULL := +WHITE := $(NULL) $(NULL) +COLON := : +COMPOSE_FILE = $(subst $(WHITE),$(COLON),$(COMPOSE_FILES)) + .EXPORT_ALL_VARIABLES: clean: @@ -45,13 +57,19 @@ build: $(MAKE) -C _modules/kitodo-production-docker build start: - docker-compose --env-file .env -f docker-compose.yml -f _modules/kitodo-production-docker/kitodo/docker-compose.yml -f _modules/ocrd_controller/docker-compose.yml -f docker-compose.kitodo-app.override.yml -f docker-compose.ocrd_controller.override.yml up --build -d + docker-compose up --build -d down: - docker-compose --env-file .env -f docker-compose.yml -f _modules/kitodo-production-docker/kitodo/docker-compose.yml -f _modules/ocrd_controller/docker-compose.yml -f docker-compose.kitodo-app.override.yml -f docker-compose.ocrd_controller.override.yml down + docker-compose down stop: - docker-compose --env-file .env -f docker-compose.yml -f _modules/kitodo-production-docker/kitodo/docker-compose.yml -f _modules/ocrd_controller/docker-compose.yml -f docker-compose.kitodo-app.override.yml -f docker-compose.ocrd_controller.override.yml stop + docker-compose stop + +config: + docker-compose config + +status: + docker-compose ps define HELP cat <<"EOF" @@ -60,15 +78,18 @@ Targets: - start run docker-compose up - down stop & rm docker-compose up - stop stops docker-compose up + - config dump all the composed files + - status list running containers Variables: - CONTROLLER_ENV_UID user id to use on the OCR-D Controller (default: $(CONTROLLER_ENV_UID)) - CONTROLLER_ENV_GID group id to use on the OCR-D Controller (default: $(CONTROLLER_ENV_GID)) - MANAGER_ENV_UID user id to use on the OCR-D Manager (default: $(MANAGER_ENV_UID)) - MANAGER_ENV_GID group id to use on the OCR-D Manager (default: $(MANAGER_ENV_GID)) + - MODE if 'managed', also starts/stops OCR-D Controller here (default: $(MODE)) EOF endef export HELP help: ; @eval "$$HELP" -.PHONY: clean build start down help +.PHONY: clean build start down config help diff --git a/README.md b/README.md index 87970c73..359a20fc 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,20 @@ * [Prerequisites](#prerequisites) * [Setup](#setup) + * [Credentials and test database](#credentials-and-test-database) + * [OCR-D models](#ocr-d-models) * [Usage](#usage) * [Docker Compose](#docker-compose) + * [Preparation](#preparation) + * [Starting](#starting) + * [Stopping and removing](#stopping-and-removing) + * [Stopping](#stopping) + * [Dumping](#dumping) + * [Status](#status) + * [Configuration](#configuration) * [Kitodo](#kitodo) - * [Futher informations and repositories](#futher-informations-and-repositories) + * [Execute OCR script step](#execute-ocr-script-step) + * [References](#references) ## Prerequisites @@ -19,11 +29,15 @@ https://docs.docker.com/compose/install/ ### Git -After cloning our repository following command to clone submodules can be used: +Either clone recursively in the first place: + + git clone --recurse-submodules https://github.com/markusweigelt/kitodo_production_ocrd + cd kitodo_production_ocrd + +Or, after cloning and entering the repository normally, clone all submodules: + + git submodule update --init --recursive -``` - git submodule update --init --recursive -``` ## Setup @@ -31,7 +45,7 @@ After cloning our repository following command to clone submodules can be used: Go to the directory where you've put your checkout. -Before docker compose can be started, you must create directories to mount an SSH key pair for user authentication to OCR-D Controller and OCR-D Manager. +Before Docker Compose can be started, you must create directories to mount an SSH key pair for user authentication to OCR-D Controller and OCR-D Manager. The fastest way is to use the makefile via the following command: @@ -57,35 +71,77 @@ This could be done persistently via the filesystem, or dynamically: ### Docker Compose +#### Preparation + +Unless you want to use `make` below, export all config files into a variable, +so you won't have to type them each time: + + export COMPOSE_FILE=docker_compose.yml:docker_compose.managed.yml:docker-compose.kitodo-app.yml:docker-compose.kitodo-app.override.yml + +Or, if you have started the [OCR-D Controller](https://github.com/bertsky/ocrd_controller) externally already: + + export COMPOSE_FILE=docker_compose.yml:docker-compose.kitodo-app.yml:docker-compose.kitodo-app.override.yml + + +Otherwise (when using the Makefile), just set your mode of operation: + + export MODE=managed + +Or, if you have started the [OCR-D Controller](https://github.com/bertsky/ocrd_controller) externally already: + + export MODE=standalone + #### Starting - docker-compose -f docker-compose.yml -f ./_modules/ocrd_controller/docker-compose.yml up --build -d + docker-compose up --build -d (or equivalently:) make start - + #### Stopping and removing -Removes the stopped containers as well as any created networks. +Removes the stopped containers as well as any created networks. - docker-compose -f docker-compose.yml -f ./_modules/ocrd_controller/docker-compose.yml down + docker-compose down (or equivalently:) - make down - + make down + #### Stopping - docker-compose -f docker-compose.yml -f ./_modules/ocrd_controller/docker-compose.yml stop + docker-compose stop + +(or equivalently:) + + make stop + +#### Dumping + + docker-compose config (or equivalently:) - make stop + make config -#### Enviroment Variables +#### Status + + docker-compose ps + +(or equivalently:) + + make status + +#### Configuration + +The following environment variables must be defined. +(This can also be done in a `.env` file, syntax is shell-like.) ##### Controller + +(only relevant in `managed` mode) + | Name | Default | Description | --- | --- | --- | | CONTROLLER_IMAGE | bertsky/ocrd_controller | name of image | @@ -94,7 +150,8 @@ Removes the stopped containers as well as any created networks. | CONTROLLER_ENV_UID | 1001 | user id of ssh user | | CONTROLLER_ENV_GID | 1001 | group id of ssh user | | CONTROLLER_ENV_UMASK | 0002 | "ssh user specific permission mask | -| CONTROLLER_PORT_SSH | 8022 | Host machine port to exposed ssh port of container | +| CONTROLLER_PORT_SSH | 8022 | host-side port to exposed SSH server of container | +| CONTROLLER_DATA | ./kitodo/config_modules/metadata | data volume to mount | ##### Manager | Name | Default | Description @@ -105,27 +162,44 @@ Removes the stopped containers as well as any created networks. | MANAGER_ENV_UID | 1001 | user id of ssh user | | MANAGER_ENV_GID | 1001 | group id of ssh user | | MANAGER_ENV_UMASK | 0002 | ssh user specific permission mask | -| MANAGER_PORT_SSH | 9022 | host machine port to exposed ssh port of container | +| MANAGER_PORT_SSH | 9022 | host-side port to exposed SSH server of container | +| MANAGER_DATA | ./kitodo/config_modules/metadata | data volume to mount | + +(Currently, `MANAGER_DATA` should be the same as `CONTROLLER_DATA`) + +##### Monitor + +| Name | Default | Description +| --- | --- | --- | +| MONITOR_IMAGE | bertsky/ocrd_monitor | name of image | +| MONITOR_HOST | ocrd-monitor | name of host | +| MONITOR_PORT_WEB | 5000 | host-side port to exposed Web server of container | +| MONITOR_PORT_GTK | 8085 | host-side port to exposed Broadwayd of container | +| MONITOR_DATA | ./kitodo/config_modules/metadata | data volume to mount | -#### Further informations +(Currently, `MONITOR_DATA` should be the same as `MANAGER_DATA`) ##### Use stable environment file - -Docker compose uses the .env file as default and thus the latest state of development. If you want to use a more stable state, use .env.stable as environment file. To accomplish this, add the following parameter "--env-file" with the value ".env.stable" to the docker-compose call. -``` - docker-compose --env-file .env.stable -f docker-compose.yml -f docker-compose-controller.yml up -d -``` +Docker Compose uses the `.env` file as default and thus the latest state of development. +If you want to use a more stable state, use `.env.stable` as environment file. +To accomplish this, either add `--env-file .env.stable` to the `docker-compose` call, +or symlink it to `.env`. + + docker-compose --env-file .env.stable up -d + ##### Overwrite environment variables -Due to the permission, the UID and GID of the controller and manager in our makefile are taken from the current system. System environment variables are used by docker-compose before .env file variables. For example: +To optimally match ownership and permissions of existing files with data to be written, +the UID and GID of the Controller and Manager in our Makefile are taken from the host system. +Shell environment variables take precedence over `.env` file variables in Docker Compose. +For example: + + export CONTROLLER_ENV_UID=$(shell id -u) CONTROLLER_HOST=ocrserver + docker-compose up -d + -``` - export CONTROLLER_ENV_UID=$(shell id -u) && \ - docker-compose -f docker-compose.yml -f docker-compose-controller.yml up -d -``` - ### Kitodo Open your browser and navigate to http://localhost:8080/kitodo after OCR-D and Kitodo are started. @@ -147,7 +221,12 @@ Type following text in script field: action:runscript stepname:OCR script:OCR ``` -... and click on "Execute KitodoScript" +... and click on "Execute KitodoScript". +(This will run the simplistic Tesseract-based default workflow +asynchronously. The process status will change as soon as the job +is finished.) + +Watch `docker logs`, or browse to `$MONITOR_HOST:5000` to look under the hood. ## References diff --git a/_modules/ocrd_manager b/_modules/ocrd_manager index c1eb02f5..ab2a8ce8 160000 --- a/_modules/ocrd_manager +++ b/_modules/ocrd_manager @@ -1 +1 @@ -Subproject commit c1eb02f5b28310dccd33255efffd9d53278805a9 +Subproject commit ab2a8ce8d8c8bc5cae42dbbf72c0312fb6cd80ca diff --git a/docker-compose.kitodo-app.yml b/docker-compose.kitodo-app.yml new file mode 120000 index 00000000..bccf5e0e --- /dev/null +++ b/docker-compose.kitodo-app.yml @@ -0,0 +1 @@ +_modules/kitodo-production-docker/kitodo/docker-compose.yml \ No newline at end of file diff --git a/docker-compose.managed.yml b/docker-compose.managed.yml new file mode 100644 index 00000000..6a94805e --- /dev/null +++ b/docker-compose.managed.yml @@ -0,0 +1,12 @@ +# add if OCR-D Controller is to be managed from here, centrally + +services: + + ocrd-controller: + extends: + file: _modules/ocrd_controller/docker-compose.yml + service: ocrd-controller + + ocrd-manager: + depends_on: + - ocrd-controller diff --git a/docker-compose.ocrd_controller.override.yml b/docker-compose.ocrd_controller.override.yml deleted file mode 100644 index 29b031d6..00000000 --- a/docker-compose.ocrd_controller.override.yml +++ /dev/null @@ -1,7 +0,0 @@ -version: "3.9" -services: - - ocrd-controller: - build: - context: _modules/ocrd_controller - dockerfile: Dockerfile \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index f08fa527..7b4a82cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,36 +1,21 @@ -version: "3.9" services: ocrd-manager: - build: - context: _modules/ocrd_manager - dockerfile: Dockerfile - - image: ${MANAGER_IMAGE}:${MANAGER_IMAGE_TAG} + extends: + file: _modules/ocrd_manager/docker-compose.yml + service: ocrd-manager - hostname: ${MANAGER_HOST} + ocrd-monitor: + extends: + file: _modules/ocrd_manager/ocrd_monitor/docker-compose.yml + service: ocrd-monitor - environment: - - UID=${MANAGER_ENV_UID} - - GID=${MANAGER_ENV_GID} - - UMASK=${MANAGER_ENV_UMASK} - - "CONTROLLER=${CONTROLLER_HOST}:22" - - "ACTIVEMQ=kitodo-mq:61616" - - ports: - - ${MANAGER_PORT_SSH}:22 +# not defined here (to be replaced by docker-compose.managed.yml if needed): +# service ocrd-controller - depends_on: - - ocrd-controller - - volumes: - - type: bind - source: ./ocrd/manager/.ssh/authorized_keys # kitodo public key - target: /authorized_keys - - type: bind - source: ./ocrd/manager/.ssh/id_rsa - target: /id_rsa - - ./kitodo/config_modules/metadata:/data # metadata directory - - tty: true # docker run -t - \ No newline at end of file +# not defined here (to be included on command-line): +# services from _modules/kitodo-production-docker/ and their local overrides +# - kitodo-app +# - kitodo-db +# - kitodo-es +# - kitodo-mq