📌 Maintenance/webserver requirements

requirements/base.Makefile

@@ -8,7 +8,8 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 .PHONY: touch reqs check clean help
 .DEFAULT_GOAL := help
 
-UPGRADE_OPTION := $(if $(upgrade),--upgrade-package $(upgrade),--upgrade)
+DO_CLEAN_OR_UPGRADE:=$(if $(clean),,--upgrade)
+UPGRADE_OPTION := $(if $(upgrade),--upgrade-package $(upgrade),$(DO_CLEAN_OR_UPGRADE))
 
 
 objects = $(sort $(wildcard *.in))

requirements/tools/Dockerfile

@@ -28,8 +28,6 @@ RUN pip --no-cache-dir install --upgrade \
 
 # devenv
 RUN pip install \
-  black \
-  isort \
   pip-tools \
-  bump2version \
-  rope
+  pipreqs \
+  pipdeptree

requirements/tools/Makefile

@@ -90,9 +90,8 @@ build build-nc: ## builds tooling image ${IMAGE_NAME}
 run: build ## Runs upgrade in a container [WARNING! UNDER DEV. USE CAREFULY]
 	docker run -it \
 		--workdir="/home/$(USER)" \
-    --volume="/etc/group:/etc/group:ro" \
+		--volume="/etc/group:/etc/group:ro" \
     --volume="/etc/passwd:/etc/passwd:ro" \
-    --volume="/etc/shadow:/etc/shadow:ro" \
 		--volume=$(REPODIR):/home/$(USER) \
 		--user=$(shell id -u):$(shell id -g) \
 		--entrypoint=/bin/bash \
@@ -104,9 +103,8 @@ run: build ## Runs upgrade in a container [WARNING! UNDER DEV. USE CAREFULY]
 shell: build ## Opens shell in ${IMAGE_NAME}
 	docker run -it \
 		--workdir="/home/$(USER)/requirements/tools" \
-    --volume="/etc/group:/etc/group:ro" \
+		--volume="/etc/group:/etc/group:ro" \
     --volume="/etc/passwd:/etc/passwd:ro" \
-		--volume="/etc/shadow:/etc/shadow:ro" \
 		--volume=$(REPODIR):/home/$(USER) \
 		--user=$(shell id -u):$(shell id -g) \
 		--entrypoint=/bin/bash \

scripts/common.Makefile

@@ -153,6 +153,12 @@ version-major: ## commits version with backwards-INcompatible addition or change
 	$(_bumpversion)
 
 
+.PHONE: pip-freeze
+pip-freeze: ## dumps current environ and base.txt [diagnostics]
+	pip freeze > freeze-now.ignore.txt
+	cat requirements/_base.txt | grep -v '#' > freeze-base.ignore.txt
+
+
 #
 # SUBTASKS
 #

services/web/server/requirements/Makefile

@@ -8,7 +8,3 @@ include ../../../../requirements/base.Makefile
 
 # TODO: Check these dependencies packages_input_reqs = $(shell grep "requirements/_base.in" _base.in | awk '{print $$2}')
 # _base.txt: _base.in $(packages_input_reqs)
-
-_base.txt: _base.in _packages.in
-_packages.txt: _packages.in _base.in
-_test.txt: _base.txt _packages.txt

services/web/server/requirements/_base.in

@@ -2,18 +2,19 @@
 # Specifies third-party dependencies for 'services/web/server/src'
 #
 --constraint ../../../../requirements/constraints.txt
+--constraint ./constraints.txt
 
-# intra-repo packages CONSTRAINTS
+# intra-repo packages
 #   - Input-requirements on intra-repo packages are tested using latest updates
 #   - Added as constraints instead of requirements in order to avoid polluting base.txt
 #   - Will be installed when prod.txt or dev.txt
 #
---constraint ../../../../packages/models-library/requirements/_base.in
---constraint ../../../../packages/postgres-database/requirements/_base.in
---constraint ../../../../packages/settings-library/requirements/_base.in
+--requirement ../../../../packages/models-library/requirements/_base.in
+--requirement ../../../../packages/postgres-database/requirements/_base.in
+--requirement ../../../../packages/settings-library/requirements/_base.in
 # simcore-service-library[aiohttp]
---constraint ../../../../packages/service-library/requirements/_base.in
---constraint ../../../../packages/service-library/requirements/_aiohttp.in
+--requirement ../../../../packages/service-library/requirements/_base.in
+--requirement ../../../../packages/service-library/requirements/_aiohttp.in
 
 
 # aiohttp + extensions
@@ -31,10 +32,13 @@ gunicorn[setproctitle]
 # socket_id saved in redis does not correspond to client's sio
 python-socketio~=4.6.1
 
-# databases
+# postgres db
 aiopg[sa]
 asyncpg
 
+# i/o
+aiofiles
+
 # redis
 aioredis
 aioredlock
@@ -67,6 +71,7 @@ aiodebug
 change_case
 expiringdict
 semantic_version
+tenacity
 
 # import/export excel
 parfive

services/web/server/requirements/_base.txt

@@ -9,22 +9,28 @@ aio-pika==6.8.0
 aiodebug==1.1.2
     # via
     #   -c requirements/../../../../packages/service-library/requirements/./_base.in
-    #   -c requirements/../../../../packages/service-library/requirements/_base.in
+    #   -r requirements/../../../../packages/service-library/requirements/_base.in
+    #   -r requirements/_base.in
+aiofiles==0.8.0
+    # via
+    #   -c requirements/../../../../packages/service-library/requirements/./_base.in
+    #   -r requirements/../../../../packages/service-library/requirements/_base.in
     #   -r requirements/_base.in
 aiohttp==3.7.4.post0
     # via
     #   -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/./../../../requirements/constraints.txt
-    #   -c requirements/../../../../packages/service-library/requirements/_aiohttp.in
     #   -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../requirements/constraints.txt
+    #   -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
     #   -r requirements/_base.in
     #   aiohttp-jinja2
     #   aiohttp-security
     #   aiohttp-session
     #   aiohttp-swagger
+    #   aiozipkin
     #   parfive
 aiohttp-jinja2==1.4.2
     # via -r requirements/_base.in
@@ -36,7 +42,7 @@ aiohttp-swagger==1.0.15
     # via -r requirements/_base.in
 aiopg==1.2.1
     # via
-    #   -c requirements/../../../../packages/service-library/requirements/_aiohttp.in
+    #   -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
     #   -r requirements/_base.in
 aioredis==1.3.1
     # via
@@ -54,6 +60,10 @@ aiormq==3.3.1
     # via aio-pika
 aiosmtplib==1.1.6
     # via -r requirements/_base.in
+aiozipkin==1.1.1
+    # via -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
+alembic==1.7.5
+    # via -r requirements/../../../../packages/postgres-database/requirements/_base.in
 async-timeout==3.0.1
     # via
     #   aiohttp
@@ -65,15 +75,19 @@ attrs==20.3.0
     # via
     #   -c requirements/../../../../packages/service-library/requirements/././constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/./constraints.txt
-    #   -c requirements/../../../../packages/service-library/requirements/_aiohttp.in
+    #   -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
     #   aiohttp
     #   aioredlock
+    #   jsonschema
+    #   openapi-core
 cffi==1.14.5
     # via cryptography
 change-case==0.5.2
     # via -r requirements/_base.in
 chardet==4.0.0
     # via aiohttp
+click==8.0.3
+    # via typer
 cryptography==3.4.7
     # via
     #   -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -98,12 +112,27 @@ hiredis==2.0.0
     # via aioredis
 idna==2.10
     # via
-    #   -c requirements/../../../../packages/models-library/requirements/_base.in
-    #   -c requirements/../../../../packages/postgres-database/requirements/_base.in
     #   -c requirements/../../../../packages/service-library/requirements/././constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/./constraints.txt
+    #   -r requirements/../../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../../packages/postgres-database/requirements/_base.in
     #   email-validator
     #   yarl
+importlib-metadata==4.10.0
+    # via alembic
+importlib-resources==5.4.0 ; python_version < "3.9"
+    # via
+    #   -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../../requirements/constraints.txt
+    #   alembic
+isodate==0.6.1
+    # via
+    #   openapi-core
+    #   openapi-schema-validator
 jinja-app-loader==1.0.2
     # via -r requirements/_base.in
 jinja2==2.11.3
@@ -120,14 +149,37 @@ json2html==1.3.0
     # via -r requirements/_base.in
 jsondiff==1.3.0
     # via -r requirements/_base.in
+jsonschema==3.2.0
+    # via
+    #   -c requirements/../../../../packages/service-library/requirements/././constraints.txt
+    #   -c requirements/../../../../packages/service-library/requirements/./constraints.txt
+    #   -c requirements/./constraints.txt
+    #   -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
+    #   openapi-schema-validator
+    #   openapi-spec-validator
+lazy-object-proxy==1.4.3
+    # via
+    #   -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
+    #   openapi-core
+mako==1.1.6
+    # via alembic
 markupsafe==1.1.1
     # via
     #   aiohttp-swagger
     #   jinja2
+    #   mako
 multidict==5.1.0
     # via
     #   aiohttp
     #   yarl
+openapi-core==0.12.0
+    # via -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
+openapi-schema-validator==0.1.6
+    # via
+    #   -c requirements/./constraints.txt
+    #   openapi-spec-validator
+openapi-spec-validator==0.3.1
+    # via openapi-core
 openpyxl==3.0.7
     # via -r requirements/_base.in
 orjson==3.5.3
@@ -138,6 +190,8 @@ parfive==1.5.1
     # via -r requirements/_base.in
 passlib==1.7.4
     # via -r requirements/_base.in
+prometheus-client==0.12.0
+    # via -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
 psycopg2-binary==2.9.1
     # via
     #   aiopg
@@ -147,16 +201,25 @@ pycparser==2.20
 pydantic==1.8.2
     # via
     #   -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../../packages/models-library/requirements/_base.in
     #   -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/./../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/./_base.in
-    #   -c requirements/../../../../packages/service-library/requirements/_base.in
     #   -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../../packages/settings-library/requirements/_base.in
     #   -c requirements/../../../../requirements/constraints.txt
+    #   -r requirements/../../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../../packages/service-library/requirements/_base.in
+    #   -r requirements/../../../../packages/settings-library/requirements/_base.in
     #   -r requirements/_base.in
+pyinstrument==3.4.2
+    # via
+    #   -c requirements/../../../../packages/service-library/requirements/./_base.in
+    #   -c requirements/./constraints.txt
+    #   -r requirements/../../../../packages/service-library/requirements/_base.in
+pyinstrument-cext==0.2.4
+    # via pyinstrument
+pyrsistent==0.18.0
+    # via jsonschema
 python-engineio==3.14.2
     # via python-socketio
 python-magic==0.4.24
@@ -172,29 +235,43 @@ pyyaml==5.4.1
     #   -c requirements/../../../../packages/service-library/requirements/././constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/./_base.in
     #   -c requirements/../../../../packages/service-library/requirements/./constraints.txt
-    #   -c requirements/../../../../packages/service-library/requirements/_base.in
     #   -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../requirements/constraints.txt
+    #   -r requirements/../../../../packages/service-library/requirements/_base.in
     #   aiohttp-swagger
+    #   openapi-spec-validator
     #   trafaret-config
 semantic-version==2.8.5
     # via -r requirements/_base.in
 setproctitle==1.2.2
     # via gunicorn
 six==1.16.0
     # via
+    #   isodate
+    #   jsonschema
+    #   openapi-core
+    #   openapi-schema-validator
+    #   openapi-spec-validator
     #   python-engineio
     #   python-socketio
 sqlalchemy==1.3.24
     # via
     #   -c requirements/../../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../../packages/postgres-database/requirements/_base.in
     #   -c requirements/../../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/service-library/requirements/./../../../requirements/constraints.txt
     #   -c requirements/../../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../../requirements/constraints.txt
+    #   -r requirements/../../../../packages/postgres-database/requirements/_base.in
     #   aiopg
+    #   alembic
+strict-rfc3339==0.7
+    # via openapi-core
+tenacity==8.0.1
+    # via
+    #   -c requirements/../../../../packages/service-library/requirements/./_base.in
+    #   -r requirements/../../../../packages/service-library/requirements/_base.in
+    #   -r requirements/_base.in
 tqdm==4.61.1
     # via parfive
 trafaret==2.1.0
@@ -203,20 +280,28 @@ trafaret==2.1.0
     #   trafaret-config
 trafaret-config==2.0.2
     # via -r requirements/_base.in
+typer==0.4.0
+    # via -r requirements/../../../../packages/settings-library/requirements/_base.in
 typing-extensions==3.10.0.2
     # via
     #   aiohttp
     #   aiohttp-jinja2
     #   pydantic
 ujson==4.0.2
     # via aiohttp-swagger
+werkzeug==2.0.2
+    # via -r requirements/../../../../packages/service-library/requirements/_aiohttp.in
 yarl==1.5.1
     # via
-    #   -c requirements/../../../../packages/postgres-database/requirements/_base.in
+    #   -r requirements/../../../../packages/postgres-database/requirements/_base.in
     #   -r requirements/_base.in
     #   aio-pika
     #   aiohttp
     #   aiormq
+zipp==3.7.0
+    # via
+    #   importlib-metadata
+    #   importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools