✨ Enhances service-integration library (#2615)

.gitignore

@@ -114,7 +114,6 @@ node_modules/
 .vscode/*
 !.vscode/*.template.json
 
-
 # manual overrides
 services/docker-compose.override.yml
 
@@ -169,3 +168,6 @@ dask*-space/
 
 # produced when mounting volumes on docker and pip installing
 .local/
+
+# Makefile artifacts
+packages/service-integration/compose-spec.json

Makefile

@@ -257,7 +257,7 @@ down: ## Stops and removes stack
 	# Removing generated docker compose configurations, i.e. .stack-*
 	-@rm $(wildcard .stack-*)
 	# Removing local registry if any
-	-@docker rm --force $(local_registry)
+	-@docker rm --force $(LOCAL_REGISTRY_HOSTNAME)
 
 leave: ## Forces to stop all services, networks, etc by the node leaving the swarm
 	-docker swarm leave -f
@@ -422,50 +422,55 @@ postgres-upgrade: ## initalize or upgrade postgres db to latest state
 	@$(MAKE_C) packages/postgres-database/docker upgrade
 
 
-local_registry=registry
+## LOCAL DOCKER REGISTRY (for local development only) -------------------------------
+
+LOCAL_REGISTRY_HOSTNAME := registry
+LOCAL_REGISTRY_VOLUME   := $(LOCAL_REGISTRY_HOSTNAME)
+
 .PHONY: local-registry rm-registry
 
 rm-registry: ## remove the registry and changes to host/file
-	@$(if $(shell grep "127.0.0.1 $(local_registry)" /etc/hosts),\
+	@$(if $(shell grep "127.0.0.1 $(LOCAL_REGISTRY_HOSTNAME)" /etc/hosts),\
 		echo removing entry in /etc/hosts...;\
-		sudo sed -i "/127.0.0.1 $(local_registry)/d" /etc/hosts,\
+		sudo sed -i "/127.0.0.1 $(LOCAL_REGISTRY_HOSTNAME)/d" /etc/hosts,\
 		echo /etc/hosts is already cleaned)
-	@$(if $(shell grep "{\"insecure-registries\": \[\"$(local_registry):5000\"\]}" /etc/docker/daemon.json),\
+	@$(if $(shell grep "{\"insecure-registries\": \[\"$(LOCAL_REGISTRY_HOSTNAME):5000\"\]}" /etc/docker/daemon.json),\
 		echo removing entry in /etc/docker/daemon.json...;\
-		sudo sed -i '/{"insecure-registries": \["$(local_registry):5000"\]}/d' /etc/docker/daemon.json;,\
+		sudo sed -i '/{"insecure-registries": \["$(LOCAL_REGISTRY_HOSTNAME):5000"\]}/d' /etc/docker/daemon.json;,\
 		echo /etc/docker/daemon.json is already cleaned)
-
-
-
+	# removing container and volume
+	-docker rm --force $(LOCAL_REGISTRY_HOSTNAME)
+	-docker volume rm $(LOCAL_REGISTRY_VOLUME)
 
 local-registry: .env ## creates a local docker registry and configure simcore to use it (NOTE: needs admin rights)
-	@$(if $(shell grep "127.0.0.1 $(local_registry)" /etc/hosts),,\
-					echo configuring host file to redirect $(local_registry) to 127.0.0.1; \
-					sudo echo 127.0.0.1 $(local_registry) | sudo tee -a /etc/hosts;\
+	@$(if $(shell grep "127.0.0.1 $(LOCAL_REGISTRY_HOSTNAME)" /etc/hosts),,\
+					echo configuring host file to redirect $(LOCAL_REGISTRY_HOSTNAME) to 127.0.0.1; \
+					sudo echo 127.0.0.1 $(LOCAL_REGISTRY_HOSTNAME) | sudo tee -a /etc/hosts;\
 					echo done)
 	@$(if $(shell grep "{\"insecure-registries\": \[\"registry:5000\"\]}" /etc/docker/daemon.json),,\
 					echo configuring docker engine to use insecure local registry...; \
-					sudo echo {\"insecure-registries\": [\"$(local_registry):5000\"]} | sudo tee -a /etc/docker/daemon.json; \
+					sudo echo {\"insecure-registries\": [\"$(LOCAL_REGISTRY_HOSTNAME):5000\"]} | sudo tee -a /etc/docker/daemon.json; \
 					echo restarting engine...; \
 					sudo service docker restart;\
 					echo done)
 	@$(if $(shell docker ps --format="{{.Names}}" | grep registry),,\
-					echo starting registry on $(local_registry):5000...; \
+					echo starting registry on $(LOCAL_REGISTRY_HOSTNAME):5000...; \
 					docker run --detach \
 							--init \
+							--env REGISTRY_STORAGE_DELETE_ENABLED=true \
 							--publish 5000:5000 \
-							--volume $(local_registry):/var/lib/registry \
-							--name $(local_registry) \
+							--volume $(LOCAL_REGISTRY_VOLUME):/var/lib/registry \
+							--name $(LOCAL_REGISTRY_HOSTNAME) \
 							registry:2)
 
 	# WARNING: environment file .env is now setup to use local registry on port 5000 without any security (take care!)...
 	@echo REGISTRY_AUTH=False >> .env
 	@echo REGISTRY_SSL=False >> .env
-	@echo REGISTRY_PATH=$(local_registry):5000 >> .env
+	@echo REGISTRY_PATH=$(LOCAL_REGISTRY_HOSTNAME):5000 >> .env
 	@echo REGISTRY_URL=$(get_my_ip):5000 >> .env
 	@echo DIRECTOR_REGISTRY_CACHING=False >> .env
 	@echo CATALOG_BACKGROUND_TASK_REST_TIME=1 >> .env
-	# local registry set in $(local_registry):5000
+	# local registry set in $(LOCAL_REGISTRY_HOSTNAME):5000
 	# images currently in registry:
 	@sleep 3
 	curl --silent $(LOCAL_REGISTRY_HOSTNAME):5000/v2/_catalog | jq '.repositories'

packages/models-library/src/models_library/basic_types.py

@@ -2,12 +2,17 @@
 
 from pydantic import conint, constr
 
+from .basic_regex import VERSION_RE
+
 # port number range
 PortInt = conint(gt=0, lt=65535)
 
 # e.g. 'v5'
 VersionTag = constr(regex=r"^v\d$")
 
+# e.g. '1.23.11' or '2.1.0-rc2'
+VersionStr = constr(regex=VERSION_RE)
+
 # checksums
 SHA1Str = constr(regex=r"^[a-fA-F0-9]{40}$")
 MD5Str = constr(regex=r"^[a-fA-F0-9]{32}$")

packages/models-library/src/models_library/services.py

@@ -24,11 +24,18 @@
 
 # NOTE: needs to end with / !!
 SERVICE_KEY_RE = r"^(simcore)/(services)/(comp|dynamic|frontend)(/[\w/-]+)+$"
+
 DYNAMIC_SERVICE_KEY_RE = r"^(simcore)/(services)/dynamic(/[\w/-]+)+$"
+DYNAMIC_SERVICE_KEY_FORMAT = "simcore/services/dynamic/{service_name}"
+
 COMPUTATIONAL_SERVICE_KEY_RE = r"^(simcore)/(services)/comp(/[\w/-]+)+$"
+COMPUTATIONAL_SERVICE_KEY_FORMAT = "simcore/services/comp/{service_name}"
+
 KEY_RE = SERVICE_KEY_RE  # TODO: deprecate this global constant by SERVICE_KEY_RE
+
 SERVICE_NETWORK_RE = r"^([a-zA-Z0-9_-]+)$"
 
+
 PROPERTY_TYPE_RE = r"^(number|integer|boolean|string|data:([^/\s,]+/[^/\s,]+|\[[^/\s,]+/[^/\s,]+(,[^/\s]+/[^/,\s]+)*\]))$"
 PROPERTY_KEY_RE = r"^[-_a-zA-Z0-9]+$"
 

packages/models-library/tests/test_services.py

@@ -10,6 +10,8 @@
 import yaml
 from models_library.basic_regex import VERSION_RE
 from models_library.services import (
+    COMPUTATIONAL_SERVICE_KEY_FORMAT,
+    DYNAMIC_SERVICE_KEY_FORMAT,
     SERVICE_KEY_RE,
     ServiceAccessRightsAtDB,
     ServiceCommonData,
@@ -168,6 +170,24 @@ def test_service_key_regex_patterns(service_key: str, regex_pattern: str):
     assert match.group(3) in ["comp", "dynamic", "frontend"]
     assert match.group(4) is not None
 
+    # tests formatters
+    new_service_key = None
+    service_type = match.group(3)
+    service_name = match.group(4).strip(
+        "/"
+    )  # FIXME: SERVICE_KEY_RE MUST eliminate / in the last capture!!!
+    if service_type == "comp":
+        new_service_key = COMPUTATIONAL_SERVICE_KEY_FORMAT.format(
+            service_name=service_name
+        )
+    elif service_type == "dynamic":
+        new_service_key = DYNAMIC_SERVICE_KEY_FORMAT.format(service_name=service_name)
+
+    if new_service_key:
+        new_match = re.match(regex_pattern, new_service_key)
+        assert new_match
+        assert new_match.groups() == match.groups()
+
 
 @pytest.mark.parametrize(
     "model_cls",

packages/service-integration/Makefile

@@ -19,3 +19,27 @@ install-dev install-prod install-ci: _check_venv_active ## install app in develo
 tests: ## runs unit tests
 	# running unit tests
 	@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests
+
+
+
+# Auto-generation of compose-spec model
+
+compose-spec.json:
+	# Downloading schema from https://github.com/compose-spec/compose-spec
+	wget https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/$@
+	echo $(shell cat $@ | jq '."$$schema"')
+
+
+.PHONY:
+_compose_spec_model.py: ## auto-generates pydantic model for compose-specification
+	# auto-generates $@ from $<
+	datamodel-codegen  \
+		--url https://raw.githubusercontent.com/compose-spec/compose-spec/master/schema/compose-spec.json \
+		--input-file-type jsonschema \
+		--snake-case-field \
+		--use-schema-description  \
+		--aliases ALIASES.json \
+		--output $@
+	# formats and moves
+	black $@
+	mv $@ src/service_integration/$@

packages/service-integration/README.md

@@ -13,22 +13,25 @@ pip install "git+https://github.com/ITISFoundation/osparc-simcore.git@master#egg
 
 ```
 
-## ``simcore-service-integrator`` entrypoint
+## ``osparc-service-integrator`` entrypoint
 
-Commands of ``simcore-service-integrator`` CLI:
+Commands of ``osparc-service-integrator`` CLI:
 ```cmd
-$ simcore-service-integrator  --help
-Usage: simcore-service-integrator [OPTIONS] COMMAND [ARGS]...
+$ osparc-service-integrator  --help
+Usage: osparc-service-integrator [OPTIONS] COMMAND [ARGS]...
+
+  o2s2parc service integration library
 
 Options:
   --version  Show the version and exit.
   --help     Show this message and exit.
 
 Commands:
-  bump-version           Bumps target version in metadata
-  get-version            Prints to output requested version
-  run-creator            Creates a sh script that uses jq tool to retrieve...
-  update-compose-labels  Update a docker-compose file with json files in a...
+  bump-version  Bumps target version in metadata
+  compose       create docker image/runtime compose-specs from the osparc...
+  config        Creates osparc config from complete docker compose-spec
+  get-version   Prints to output requested version
+  run-creator   Creates a sh script that uses jq tool to retrieve...
 ```
 
 
@@ -40,13 +43,14 @@ A replacement for the old Makefile recipes might be:
 ```Makefile
 service.cli/run: $(METADATA)
 	# Updates adapter script from metadata in $<
-	simcore-service-integrator run-creator --metadata $< --runscript $@
+	osparc-service-integrator run-creator --metadata $< --runscript $@
 
 docker-compose-meta.yml: $(METADATA)
 	# Injects metadata from $< as labels
-	simcore-service-integrator update-compose-labels --compose $@ --metadata $<
+	osparc-service-integrator compose --metadata $< --to-spec-file $@
 
 ```
+
 ### testing plugin
 
 Created a pytest-plugin from submodule ``service_integration.pytest_plugin`` with fixtures and helper assert function.
@@ -90,8 +94,8 @@ TODO:
 
 Tools to increase versions by specifying ``patch, minor, major``.
 ```cmd
-$ simcore-service-integrator bump-version --help
-Usage: simcore-service-integrator bump-version [OPTIONS] [[integration-
+$ osparc-service-integrator bump-version --help
+Usage: osparc-service-integrator bump-version [OPTIONS] [[integration-
                                                version|version]]
 
   Bumps target version in metadata
@@ -108,16 +112,16 @@ so a replacement Makefile recipes might be
 CURRENT_VERSION := $(shell VERSION)
 
 VERSION: $(METADATA) ## creates VERSION file
-  @simcore-service-integrator get-version --metadata-file $< > $@
+  @osparc-service-integrator get-version --metadata-file $< > $@
 
 .PHONY: version-service-patch version-service-minor version-service-major
 version-service-patch version-service-minor version-service-major: $(METADATA) ## kernel/service versioning as patch
-	simcore-service-integrator bump-version --metadata-file $<  --upgrade $(subst version-service-,,$@)
+	osparc-service-integrator bump-version --metadata-file $<  --upgrade $(subst version-service-,,$@)
   $(MAKE) VERSION
 
 .PHONY: version-integration-patch version-integration-minor version-integration-major
 version-integration-patch version-integration-minor version-integration-major: $(METADATA) ## integration versioning as patch (bug fixes not affecting API/handling), minor/major (backwards-compatible/INcompatible API changes)
-	simcore-service-integrator bump-version --metadata-file $<  --upgrade $(subst version-integration-,,$@) integration-version
+	osparc-service-integrator bump-version --metadata-file $<  --upgrade $(subst version-integration-,,$@) integration-version
 
 ```
 

packages/service-integration/VERSION

@@ -1 +1 @@
-1.0.0
+1.0.1

packages/service-integration/requirements/_tools.in

@@ -3,3 +3,6 @@
 --constraint _test.txt
 
 --requirement ../../../requirements/devenv.txt
+
+# to generate compose_spec_model.py
+datamodel-code-generator[http]