diff --git a/packages/service-integration/Dockerfile b/packages/service-integration/Dockerfile index 239fef48ade..e7f1ca24cb5 100644 --- a/packages/service-integration/Dockerfile +++ b/packages/service-integration/Dockerfile @@ -3,6 +3,29 @@ FROM python:${PYTHON_VERSION}-slim-buster as base LABEL maintainer=pcrespov +RUN set -eux \ + && apt-get update \ + && apt-get install -y \ + git \ + && rm -rf /var/lib/apt/lists/* \ + # verify that the binary works + && git --version + +# simcore-user uid=8004(scu) gid=8004(scu) groups=8004(scu) +ENV SC_USER_ID=8004 \ + SC_USER_NAME=scu \ + SC_BUILD_TARGET=base \ + SC_BOOT_MODE=default + +RUN adduser \ + --uid ${SC_USER_ID} \ + --disabled-password \ + --gecos "" \ + --shell /bin/sh \ + --home /home/${SC_USER_NAME} \ + ${SC_USER_NAME} + + # Sets utf-8 encoding for Python et al ENV LANG=C.UTF-8 # Turns off writing .pyc files; superfluous on an ephemeral container. @@ -52,6 +75,8 @@ FROM base as development # NOTE: this is necessary to allow to build development images but is the same as production here FROM base as production +USER scu + COPY --from=build --chown=scu:scu ${VIRTUAL_ENV} ${VIRTUAL_ENV} # NOTE: do not activate ENV PYTHONOPTIMIZE=TRUE since excutable contains pytest code diff --git a/packages/service-integration/Makefile b/packages/service-integration/Makefile index 0556c4832e1..ad12089fa46 100644 --- a/packages/service-integration/Makefile +++ b/packages/service-integration/Makefile @@ -85,10 +85,21 @@ build build-nc: ## [docker] builds docker image of executable w/ or w/o cache docker run local/${PACKAGE_NAME}:production --version .PHONY: inspect -inspect: - docker image inspect local/${PACKAGE_NAME}:production | jq '.[0] | .RepoTags, .Config.Labels, .Architecture' +inspect: ## [docker] inspects container + docker image inspect \ + local/${PACKAGE_NAME}:production | jq '.[0] | .RepoTags, .Config.Labels, .Architecture' +_src_dir = $(if $(target),$(target),$(PWD)) + .PHONY: shell -shell: - docker run -it --entrypoint bash local/${PACKAGE_NAME}:production +shell: ## [docker] opens shell in container + docker run \ + -it \ + --volume="/etc/group:/etc/group:ro" \ + --volume="/etc/passwd:/etc/passwd:ro" \ + --user="$(shell id --user "$(USER)")":scu \ + --entrypoint bash \ + --volume "$(_src_dir)":/src \ + --workdir=/src \ + local/${PACKAGE_NAME}:production diff --git a/packages/service-integration/VERSION b/packages/service-integration/VERSION index 6d7de6e6abe..21e8796a09d 100644 --- a/packages/service-integration/VERSION +++ b/packages/service-integration/VERSION @@ -1 +1 @@ -1.0.2 +1.0.3 diff --git a/packages/service-integration/requirements/_base.txt b/packages/service-integration/requirements/_base.txt index bc45730d5f6..18663acce68 100644 --- a/packages/service-integration/requirements/_base.txt +++ b/packages/service-integration/requirements/_base.txt @@ -26,6 +26,8 @@ docker==6.0.0 # via -r requirements/_base.in email-validator==1.3.0 # via pydantic +exceptiongroup==1.0.0 + # via pytest idna==3.4 # via # email-validator @@ -42,8 +44,6 @@ packaging==21.3 # pytest pluggy==1.0.0 # via pytest -py==1.11.0 - # via pytest pydantic==1.10.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -55,7 +55,7 @@ pyparsing==3.0.9 # via packaging pyrsistent==0.18.1 # via jsonschema -pytest==7.1.3 +pytest==7.2.0 # via -r requirements/_base.in pyyaml==6.0 # via diff --git a/packages/service-integration/requirements/_test.txt b/packages/service-integration/requirements/_test.txt index 9b09328e405..1d16741f3b9 100644 --- a/packages/service-integration/requirements/_test.txt +++ b/packages/service-integration/requirements/_test.txt @@ -29,6 +29,10 @@ dill==0.3.5.1 # via pylint docopt==0.6.2 # via coveralls +exceptiongroup==1.0.0 + # via + # -c requirements/_base.txt + # pytest idna==3.4 # via # -c requirements/_base.txt @@ -54,17 +58,13 @@ pluggy==1.0.0 # via # -c requirements/_base.txt # pytest -py==1.11.0 - # via - # -c requirements/_base.txt - # pytest pylint==2.15.4 # via -r requirements/_test.in pyparsing==3.0.9 # via # -c requirements/_base.txt # packaging -pytest==7.1.3 +pytest==7.2.0 # via # -c requirements/_base.txt # -r requirements/_test.in diff --git a/packages/service-integration/scripts/ooil.bash b/packages/service-integration/scripts/ooil.bash new file mode 100755 index 00000000000..c664737a869 --- /dev/null +++ b/packages/service-integration/scripts/ooil.bash @@ -0,0 +1,27 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ + +set -o errexit +set -o nounset +set -o pipefail +IFS=$'\n\t' + + +IMAGE_NAME="${DOCKER_REGISTRY:-itisfoundation}/service-integration:${DOCKER_IMAGE_TAG:-master-github-latest}" +WORKDIR="$(pwd)" + +run() { + docker run \ + -it \ + --rm \ + --volume="/etc/group:/etc/group:ro" \ + --volume="/etc/passwd:/etc/passwd:ro" \ + --user="$(id --user "$USER")":"$(id --group "$USER")" \ + --volume "$WORKDIR":/src \ + --workdir=/src \ + "$IMAGE_NAME" \ + "$@" +} + + +run "$@" diff --git a/packages/service-integration/setup.cfg b/packages/service-integration/setup.cfg index 1d3f961714d..af7998eb1a3 100644 --- a/packages/service-integration/setup.cfg +++ b/packages/service-integration/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.2 +current_version = 1.0.3 commit = True message = service-integration version: {current_version} → {new_version} tag = False diff --git a/packages/service-integration/setup.py b/packages/service-integration/setup.py index 0c90b903958..b891719c58c 100644 --- a/packages/service-integration/setup.py +++ b/packages/service-integration/setup.py @@ -70,6 +70,12 @@ def read_reqs(reqs_path: Path) -> set[str]: packages=find_packages(where="src"), package_dir={"": "src"}, include_package_data=True, + package_data={ + "": [ + "service/tests/**/*.py", + "service/tests/unit/*.py", + ] + }, test_suite="tests", tests_require=TEST_REQUIREMENTS, extras_require={}, diff --git a/packages/service-integration/src/service_integration/cli.py b/packages/service-integration/src/service_integration/cli.py index d9c5f6d6c20..50dd85eb035 100644 --- a/packages/service-integration/src/service_integration/cli.py +++ b/packages/service-integration/src/service_integration/cli.py @@ -6,7 +6,7 @@ import typer from ._meta import __version__ -from .commands import compose, config, metadata, run_creator +from .commands import compose, config, metadata, run_creator, test from .settings import AppSettings app = typer.Typer() @@ -54,6 +54,7 @@ def main( # new app.command("compose")(compose.main) app.command("config")(config.main) +app.command("test")(test.main) # legacy app.command("bump-version")(metadata.bump_version) app.command("get-version")(metadata.get_version) diff --git a/packages/service-integration/src/service_integration/commands/compose.py b/packages/service-integration/src/service_integration/commands/compose.py index 80e9b7e105e..ae68a660df2 100644 --- a/packages/service-integration/src/service_integration/commands/compose.py +++ b/packages/service-integration/src/service_integration/commands/compose.py @@ -34,13 +34,19 @@ def _run_git(*args) -> str: def _run_git_or_empty_string(*args) -> str: try: return _run_git(*args) + except FileNotFoundError as err: + error_console.print( + "WARNING: Defaulting label to emtpy string", + "since git is not installed or cannot be executed:", + err, + ) except subprocess.CalledProcessError as err: error_console.print( "WARNING: Defaulting label to emtpy string", "due to:", err.stderr, ) - return "" + return "" def create_docker_compose_image_spec( diff --git a/packages/service-integration/src/service_integration/commands/test.py b/packages/service-integration/src/service_integration/commands/test.py new file mode 100644 index 00000000000..b08c5a85c03 --- /dev/null +++ b/packages/service-integration/src/service_integration/commands/test.py @@ -0,0 +1,21 @@ +from pathlib import Path + +import rich +import typer + +from ..service import pytest_runner + + +def main( + service_dir: Path = typer.Argument( + ..., help="Root directory of the service under test" + ), +): + """Runs tests against service directory""" + + if not service_dir.exists(): + raise typer.BadParameter("Invalid path to service directory") + + rich.print(f"Testing '{service_dir.resolve()}' ...") + error_code = pytest_runner.main(service_dir=service_dir, extra_args=[]) + raise typer.Exit(code=error_code) diff --git a/packages/service-integration/src/service_integration/pytest_plugin/__init__.py b/packages/service-integration/src/service_integration/pytest_plugin/__init__.py index e69de29bb2d..3186eb70fed 100644 --- a/packages/service-integration/src/service_integration/pytest_plugin/__init__.py +++ b/packages/service-integration/src/service_integration/pytest_plugin/__init__.py @@ -0,0 +1,22 @@ +import warnings + +warnings.warn( + f"{__name__} is deprecated for cookiecutter-osparc-service>0.4.Use directoy test CLI instead." + "See https://github.com/ITISFoundation/cookiecutter-osparc-service/releases/tag/v0.4.0", + DeprecationWarning, +) + + +def pytest_addoption(parser): + group = parser.getgroup("service-integration") + group.addoption( + "--service-dir", + action="store", + help="Base directory for target service", + ) + + group.addoption( + "--metadata", + action="store", + help="metadata yaml configuration file", + ) diff --git a/packages/service-integration/src/service_integration/pytest_plugin/folder_structure.py b/packages/service-integration/src/service_integration/pytest_plugin/folder_structure.py index 714a391e4eb..dc1e57fbee4 100644 --- a/packages/service-integration/src/service_integration/pytest_plugin/folder_structure.py +++ b/packages/service-integration/src/service_integration/pytest_plugin/folder_structure.py @@ -1,17 +1,22 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable -import sys from pathlib import Path import pytest -current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - @pytest.fixture(scope="session") -def project_slug_dir() -> Path: - raise NotImplementedError("Override fixture 'project_slug_dir' REQUIRED") +def project_slug_dir(request: pytest.FixtureRequest) -> Path: + try: + root_dir = Path(request.config.getoption("--service-dir")) + except TypeError: + pytest.fail("--service-dir is not set") + + assert isinstance(root_dir, Path) + assert root_dir.exists() + assert any(root_dir.glob(".osparc")) + return root_dir @pytest.fixture(scope="session") @@ -20,20 +25,6 @@ def project_name(project_slug_dir: Path) -> str: return project_slug_dir.name -@pytest.fixture(scope="session") -def src_dir(project_slug_dir: Path) -> Path: - _src_dir = project_slug_dir / "src" - assert _src_dir.exists() - return _src_dir - - -@pytest.fixture(scope="session") -def tests_dir(project_slug_dir: Path) -> Path: - _tests_dir = project_slug_dir / "tests" - assert _tests_dir.exists() - return _tests_dir - - @pytest.fixture(scope="session") def validation_dir(project_slug_dir: Path) -> Path: validation_dir = project_slug_dir / "validation" @@ -42,22 +33,13 @@ def validation_dir(project_slug_dir: Path) -> Path: @pytest.fixture(scope="session") -def tools_dir(project_slug_dir: Path) -> Path: - tools_dir = project_slug_dir / "tools" - assert tools_dir.exists() - return tools_dir - - -@pytest.fixture(scope="session") -def docker_dir(project_slug_dir: Path) -> Path: - docker_dir = project_slug_dir / "docker" - assert docker_dir.exists() - return docker_dir - +def metadata_file(project_slug_dir: Path, request: pytest.FixtureRequest) -> Path: + try: + metadata_file = Path(request.config.getoption("--metadata")) + except TypeError: + metadata_file = project_slug_dir / "metadata" / "metadata.yml" -@pytest.fixture(scope="session") -def metadata_file(project_slug_dir: Path) -> Path: - metadata_file = project_slug_dir / "metadata" / "metadata.yml" + assert isinstance(metadata_file, Path) assert metadata_file.exists() return metadata_file diff --git a/packages/service-integration/src/service_integration/service/__init__.py b/packages/service-integration/src/service_integration/service/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/service-integration/src/service_integration/service/pytest_runner.py b/packages/service-integration/src/service_integration/service/pytest_runner.py new file mode 100644 index 00000000000..9626c4d7be3 --- /dev/null +++ b/packages/service-integration/src/service_integration/service/pytest_runner.py @@ -0,0 +1,48 @@ +import logging +import sys +import tempfile +from pathlib import Path +from typing import Optional + +import pytest + +CURRENT_DIR = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent +TESTS_DIR = CURRENT_DIR / "tests" + +logger = logging.getLogger(__name__) + + +def main( + service_dir: Path, *, debug: bool = False, extra_args: Optional[list[str]] = None +) -> int: + + pytest_args = [ + # global cache options + "--cache-clear", + f"--override-ini=cache_dir={tempfile.gettempdir()}/.pytest_cache__service_integration", + # tests + f"{TESTS_DIR}", + # custom options + f"--service-under-test-dir={service_dir}", + ] + + if debug: + pytest_args += ["-vv", "--log-level=DEBUG", "--pdb"] + + if extra_args: + pytest_args += extra_args + + logger.debug("Running 'pytest %s'", " ".join(pytest_args)) + exit_code = pytest.main(pytest_args) + logger.debug("exit with code=%d", exit_code) + return exit_code + + +if __name__ == "__main__": + # Entrypoint for stand-alone 'service_integration/service' package + sys.exit( + main( + service_dir=Path(sys.argv[1]), + extra_args=sys.argv[2:], + ) + ) diff --git a/packages/service-integration/src/service_integration/service/tests/conftest.py b/packages/service-integration/src/service_integration/service/tests/conftest.py new file mode 100644 index 00000000000..dfc649a2297 --- /dev/null +++ b/packages/service-integration/src/service_integration/service/tests/conftest.py @@ -0,0 +1,25 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +from pathlib import Path + +import pytest + + +def pytest_addoption(parser: pytest.Parser): + parser.addoption( + "--service-under-test-dir", help="Root directory of the service under test" + ) + + +@pytest.fixture +def service_under_test_dir(pytestconfig: pytest.Config) -> Path: + """Base directory of the service under test (--service-under-test-dir)""" + try: + dir_path = Path(pytestconfig.getoption("--service-under-test-dir")) + except TypeError: + pytest.fail("Invalid path --service-under-test-dir") + assert dir_path.exists() + return dir_path diff --git a/packages/service-integration/src/service_integration/service/tests/unit/test_service_folders_layout.py b/packages/service-integration/src/service_integration/service/tests/unit/test_service_folders_layout.py new file mode 100644 index 00000000000..916a852e1f0 --- /dev/null +++ b/packages/service-integration/src/service_integration/service/tests/unit/test_service_folders_layout.py @@ -0,0 +1,18 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + +from pathlib import Path + + +def test_minimal_folder_layout(service_under_test_dir: Path): + assert service_under_test_dir.exists() + + # has osparc folder + assert any(service_under_test_dir.glob(".osparc/**/metadata.yml")) + + # has validation folder # TODO: define path in .osparc?? + assert (service_under_test_dir / "validation").exists() + assert (service_under_test_dir / "validation" / "input").exists() + assert (service_under_test_dir / "validation" / "output").exists() diff --git a/services/director/src/simcore_service_director/producer.py b/services/director/src/simcore_service_director/producer.py index 471041e5b88..a34f06508fc 100644 --- a/services/director/src/simcore_service_director/producer.py +++ b/services/director/src/simcore_service_director/producer.py @@ -1120,8 +1120,10 @@ async def stop_service(app: web.Application, node_uuid: str, save_state: bool) - except ClientError as err: log.warning( - "Could not save state because {service_host_name} is unreachable [{err}]." - "Resuming stop_service." + "Could not save state because %s is unreachable [%s]." + "Resuming stop_service.", + service_host_name, + err ) # remove the services