-
Notifications
You must be signed in to change notification settings - Fork 27
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
✨dynamic-sidecar limits all spawned containers (⚠️ devops) (#2988)
* refactor * refactor * adding resoruce limits for containers * fix pylint * added tests for changes * skip non possible non osparc services * refactore moved CatalogSettings to settings-library * extended resoruces * FastAPI now correctly converts DictModel * outsorced to servicelib * extended get resources for service * fix readme links * starting services now requires passing in resources * no longer try this * update example * SchedulerData contains ServiceResources * update proxy limits and reservations * update settings beore use * using incoming resources * fixed tests * fixed tests * no longer requires async keyword * fixed 3 failing tests * fixed broken tests webserver/unit/02 * fix tests * renamed Resoruces to ResourcesDict * adds tests for service_settings_labels and fixes old code * pylint formatting * fix broken tests * reverting timeout * fix broken test * fix pylint * added todo refactor * fix very slow test * bumping limits * removed test * rfactor and moved limits to correct place * makr test flaky * using proper error formatting * dropping multiple prefix * dy-sidecar has always a minimum limit of 1 CPU * refactor after merge * refactor * removing catalogsettings form director-v2 * renaming * removed outdated comment * refactor adding minimum resources for dy-sidecar * codestyle * fix failing test * fix endpoint * removed undesired * drop already fixed in different PR * catalog can now reply with resoruces for external services * fixed label fetching error not being caputred * codestyle * fixed integration tests * @pcrespov @sanderegg removed DictModel * simplified constructor and refactor tests * removing test from fixture name * refactor trying to reduce cognitive complexity * refactor names and groupings * refactor tests * refactor * fixed note * added note for future use * Git hk add limits to containers (#31) * minor * undo minor * redesign of the resources layout * catalog endpoin fixed for api-server Co-authored-by: Andrei Neagu <[email protected]> Co-authored-by: Odei Maiz <[email protected]>
- Loading branch information
3 people
authored
May 20, 2022
1 parent
e437f88
commit a501cc6
Showing
88 changed files
with
1,528 additions
and
505 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| from pydantic import constr | ||
|
|
||
| DOCKER_IMAGE_KEY_RE = r"[\w/-]+" | ||
| DOCKER_IMAGE_VERSION_RE = r"[\w/.]+" | ||
|
|
||
| DockerImageKey = constr(regex=DOCKER_IMAGE_KEY_RE) | ||
| DockerImageVersion = constr(regex=DOCKER_IMAGE_VERSION_RE) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
111 changes: 111 additions & 0 deletions
111
packages/models-library/tests/test_service_resources.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| # pylint:disable=unused-variable | ||
| # pylint:disable=unused-argument | ||
| # pylint:disable=redefined-outer-name | ||
|
|
||
| from typing import Any | ||
|
|
||
| import pytest | ||
| from models_library.services_resources import ( | ||
| DockerComposeServiceName, | ||
| DockerImage, | ||
| ImageResources, | ||
| ResourcesDict, | ||
| ResourceValue, | ||
| ServiceResourcesDict, | ||
| ServiceResourcesDictHelpers, | ||
| ) | ||
| from pydantic import parse_obj_as | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "example", | ||
| ( | ||
| "simcore/services/dynamic/the:latest", | ||
| "simcore/services/dynamic/nice-service:v1.0.0", | ||
| "a/docker-hub/image:1.0.0", | ||
| "traefik:v1.0.0", | ||
| "traefik:v1.0.0@somehash", | ||
| ), | ||
| ) | ||
| def test_compose_image(example: str) -> None: | ||
| parse_obj_as(DockerImage, example) | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def resources_dict() -> ResourcesDict: | ||
| return parse_obj_as( | ||
| ResourcesDict, ImageResources.Config.schema_extra["example"]["resources"] | ||
| ) | ||
|
|
||
|
|
||
| @pytest.fixture | ||
| def compose_image() -> DockerImage: | ||
| return parse_obj_as(DockerImage, "image:latest") | ||
|
|
||
|
|
||
| def _ensure_resource_value_is_an_object(data: ResourcesDict) -> None: | ||
| assert type(data) == dict | ||
| print(data) | ||
| for entry in data.values(): | ||
| entry: ResourceValue = entry | ||
| assert entry.limit | ||
| assert entry.reservation | ||
|
|
||
|
|
||
| def test_resources_dict_parsed_as_expected(resources_dict: ResourcesDict) -> None: | ||
| _ensure_resource_value_is_an_object(resources_dict) | ||
|
|
||
|
|
||
| def test_image_resources_parsed_as_expected() -> None: | ||
| result: ImageResources = ImageResources.parse_obj( | ||
| ImageResources.Config.schema_extra["example"] | ||
| ) | ||
| _ensure_resource_value_is_an_object(result.resources) | ||
| assert type(result) == ImageResources | ||
|
|
||
| result: ImageResources = parse_obj_as( | ||
| ImageResources, ImageResources.Config.schema_extra["example"] | ||
| ) | ||
| assert type(result) == ImageResources | ||
| _ensure_resource_value_is_an_object(result.resources) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "example", ServiceResourcesDictHelpers.Config.schema_extra["examples"] | ||
| ) | ||
| def test_service_resource_parsed_as_expected( | ||
| example: dict[DockerComposeServiceName, Any], compose_image: DockerImage | ||
| ) -> None: | ||
| def _assert_service_resources_dict( | ||
| service_resources_dict: ServiceResourcesDict, | ||
| ) -> None: | ||
| assert type(service_resources_dict) == dict | ||
|
|
||
| print(service_resources_dict) | ||
| for _, image_resources in service_resources_dict.items(): | ||
| _ensure_resource_value_is_an_object(image_resources.resources) | ||
|
|
||
| service_resources_dict: ServiceResourcesDict = parse_obj_as( | ||
| ServiceResourcesDict, example | ||
| ) | ||
| _assert_service_resources_dict(service_resources_dict) | ||
|
|
||
| for image_resources in example.values(): | ||
| service_resources_dict_from_single_service = ( | ||
| ServiceResourcesDictHelpers.create_from_single_service( | ||
| image=compose_image, | ||
| resources=ImageResources.parse_obj(image_resources).resources, | ||
| ) | ||
| ) | ||
| _assert_service_resources_dict(service_resources_dict_from_single_service) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "example", ServiceResourcesDictHelpers.Config.schema_extra["examples"] | ||
| ) | ||
| def test_create_jsonable_dict(example: dict[DockerComposeServiceName, Any]) -> None: | ||
| service_resources_dict: ServiceResourcesDict = parse_obj_as( | ||
| ServiceResourcesDict, example | ||
| ) | ||
| result = ServiceResourcesDictHelpers.create_jsonable(service_resources_dict) | ||
| assert example == result |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| import yaml | ||
|
|
||
|
|
||
| # Notes on below env var names: | ||
| # - SIMCORE_REGISTRY will be replaced by the url of the simcore docker registry | ||
| # deployed inside the platform | ||
| # - SERVICE_VERSION will be replaced by the version of the service | ||
| # to which this compos spec is attached | ||
| # Example usage in docker compose: | ||
| # image: ${SIMCORE_REGISTRY}/${DOCKER_IMAGE_NAME}-dynamic-sidecar-compose-spec:${SERVICE_VERSION} | ||
| MATCH_SERVICE_VERSION = "${SERVICE_VERSION}" | ||
| MATCH_SIMCORE_REGISTRY = "${SIMCORE_REGISTRY}" | ||
| MATCH_IMAGE_START = f"{MATCH_SIMCORE_REGISTRY}/" | ||
| MATCH_IMAGE_END = f":{MATCH_SERVICE_VERSION}" | ||
|
|
||
|
|
||
| def replace_env_vars_in_compose_spec( | ||
| service_spec: "ComposeSpecLabel", | ||
| *, | ||
| replace_simcore_registry: str, | ||
| replace_service_version: str, | ||
| ) -> str: | ||
| """ | ||
| replaces all special env vars inside docker-compose spec | ||
| returns a stringified version | ||
| """ | ||
|
|
||
| stringified_service_spec = yaml.safe_dump(service_spec) | ||
|
|
||
| # NOTE: could not use `string.Template` here because the test will | ||
| # fail since `${DISPLAY}` cannot be replaced, and we do not want | ||
| # it to be replaced at this time. If this method is changed | ||
| # the test suite should always pass without changes. | ||
| stringified_service_spec = stringified_service_spec.replace( | ||
| MATCH_SIMCORE_REGISTRY, replace_simcore_registry | ||
| ) | ||
| stringified_service_spec = stringified_service_spec.replace( | ||
| MATCH_SERVICE_VERSION, replace_service_version | ||
| ) | ||
| return stringified_service_spec |
Oops, something went wrong.