⬆️ Maintenance/update aioredis (#2898)

packages/models-library/src/models_library/projects_state.py

@@ -79,14 +79,14 @@ class Config:
 
     @validator("owner", pre=True, always=True)
     @classmethod
-    def check_not_null(v, values):
+    def check_not_null(cls, v, values):
         if values["value"] is True and v is None:
             raise ValueError("value cannot be None when project is locked")
         return v
 
     @validator("status", always=True)
     @classmethod
-    def check_status_compatible(v, values):
+    def check_status_compatible(cls, v, values):
         if values["value"] is False and v not in ["CLOSED", "OPENED"]:
             raise ValueError(
                 f"status is set to {v} and lock is set to {values['value']}!"

packages/pytest-simcore/src/pytest_simcore/redis_service.py

@@ -5,9 +5,9 @@
 import logging
 from typing import AsyncIterator, Dict, Union
 
-import aioredis
 import pytest
 import tenacity
+from redis.asyncio import Redis, from_url
 from settings_library.redis import RedisSettings
 from tenacity.before_sleep import before_sleep_log
 from tenacity.stop import stop_after_delay
@@ -34,7 +34,7 @@ async def redis_settings(
     )
     # test runner is running on the host computer
     settings = RedisSettings(REDIS_HOST=get_localhost_ip(), REDIS_PORT=int(port))
-    await wait_till_redis_responsive(settings.dsn)
+    await wait_till_redis_responsive(settings.dsn_resources)
 
     return settings
 
@@ -56,15 +56,29 @@ def redis_service(
 @pytest.fixture(scope="function")
 async def redis_client(
     redis_settings: RedisSettings,
-) -> AsyncIterator[aioredis.Redis]:
+) -> AsyncIterator[Redis]:
     """Creates a redis client to communicate with a redis service ready"""
-    client = await aioredis.create_redis_pool(redis_settings.dsn, encoding="utf-8")
+    client = from_url(
+        redis_settings.dsn_resources, encoding="utf-8", decode_responses=True
+    )
 
     yield client
 
     await client.flushall()
-    client.close()
-    await client.wait_closed()
+    await client.close()
+
+
+@pytest.fixture(scope="function")
+async def redis_locks_client(
+    redis_settings: RedisSettings,
+) -> AsyncIterator[Redis]:
+    """Creates a redis client to communicate with a redis service ready"""
+    client = from_url(redis_settings.dsn_locks, encoding="utf-8", decode_responses=True)
+
+    yield client
+
+    await client.flushall()
+    await client.close()
 
 
 # HELPERS --
@@ -77,6 +91,7 @@ async def redis_client(
     reraise=True,
 )
 async def wait_till_redis_responsive(redis_url: Union[URL, str]) -> None:
-    client = await aioredis.create_redis_pool(str(redis_url), encoding="utf-8")
-    client.close()
-    await client.wait_closed()
+    client = from_url(f"{redis_url}", encoding="utf-8", decode_responses=True)
+
+    if not await client.ping():
+        raise ConnectionError(f"{redis_url=} not available")

packages/pytest-simcore/src/pytest_simcore/services_api_mocks_for_aiohttp_clients.py

@@ -226,6 +226,13 @@ async def director_v2_service_mock(
         r"^http://[a-z\-_]*director-v2:[0-9]+/v2/dynamic_services/projects/.*/-/networks$"
     )
 
+    get_services_pattern = re.compile(
+        r"^http://[a-z\-_]*director-v2:[0-9]+/v2/dynamic_services$"
+    )
+    aioresponses_mocker.get(
+        get_services_pattern, status=web.HTTPOk.status_code, repeat=True
+    )
+
     aioresponses_mocker.post(
         create_computation_pattern,
         callback=create_computation_cb,

packages/settings-library/src/settings_library/redis.py

@@ -1,6 +1,7 @@
 from functools import cached_property
 from typing import Optional
 
+from pydantic import Field
 from pydantic.networks import RedisDsn
 from pydantic.types import SecretStr
 
@@ -18,10 +19,14 @@ class RedisSettings(BaseCustomSettings):
     REDIS_PASSWORD: Optional[SecretStr] = None
 
     # db
-    REDIS_DB: Optional[str] = "0"
+    REDIS_RESOURCES_DB: int = Field(
+        0,
+        description="typical redis DB have 16 'tables', for convenience we use this table for user resources",
+    )
+    REDIS_LOCKS_DB: int = Field(1, description="This redis table is used to put locks")
 
     @cached_property
-    def dsn(self) -> str:
+    def dsn_resources(self) -> str:
         return RedisDsn.build(
             scheme="redis",
             user=self.REDIS_USER or None,
@@ -30,5 +35,18 @@ def dsn(self) -> str:
             else None,
             host=self.REDIS_HOST,
             port=f"{self.REDIS_PORT}",
-            path=f"/{self.REDIS_DB}",
+            path=f"/{self.REDIS_RESOURCES_DB}",
+        )
+
+    @cached_property
+    def dsn_locks(self) -> str:
+        return RedisDsn.build(
+            scheme="redis",
+            user=self.REDIS_USER or None,
+            password=self.REDIS_PASSWORD.get_secret_value()
+            if self.REDIS_PASSWORD
+            else None,
+            host=self.REDIS_HOST,
+            port=f"{self.REDIS_PORT}",
+            path=f"/{self.REDIS_LOCKS_DB}",
         )

requirements/constraints.txt

@@ -22,8 +22,7 @@ urllib3>=1.26.5                               # https://github.com/advisories/GH
 # Breaking changes
 #
 
-# TODO: https://aioredis.readthedocs.io/en/latest/migration/)
-aioredis<2.0.0
+
 # with new released version 1.0.0 (https://github.com/aio-libs/aiozipkin/releases).
 # TODO: includes async features https://docs.sqlalchemy.org/en/14/changelog/migration_20.html
 sqlalchemy<2.0

services/catalog/requirements/_base.txt

@@ -10,19 +10,12 @@ aiodebug==1.1.2
     # via
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
-aiofiles==0.5.0
+aiofiles==0.8.0
     # via
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
-aioredis==1.3.1
-    # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
-    #   aiocache
+aioredis==2.0.1
+    # via aiocache
 alembic==1.7.4
     # via -r requirements/../../../packages/postgres-database/requirements/_base.in
 anyio==3.5.0
@@ -67,8 +60,6 @@ h11==0.12.0
     # via
     #   httpcore
     #   uvicorn
-hiredis==2.0.0
-    # via aioredis
 httpcore==0.14.4
     # via httpx
 httptools==0.2.0
@@ -77,9 +68,6 @@ httpx==0.21.3
     # via -r requirements/_base.in
 idna==2.10
     # via
-    #   -c requirements/../../../packages/service-library/requirements/././constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/./constraints.txt
-    #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   anyio
     #   email-validator
     #   requests
@@ -213,7 +201,9 @@ tornado==6.1
 typer==0.4.0
     # via -r requirements/../../../packages/settings-library/requirements/_base.in
 typing-extensions==3.10.0.2
-    # via pydantic
+    # via
+    #   aioredis
+    #   pydantic
 ujson==4.0.2
     # via fastapi
 urllib3==1.26.7

services/director-v2/requirements/_test.in

@@ -33,7 +33,6 @@ minio
 
 # migration due to pytest_simcore.postgres_service2
 aio_pika
-aioredis
 alembic
 bokeh
 dask-gateway-server[local]

services/director-v2/requirements/_test.txt

@@ -21,10 +21,6 @@ aiohttp==3.8.1
     #   pytest-aiohttp
 aioitertools==0.10.0
     # via aiobotocore
-aioredis==1.3.1
-    # via
-    #   -c requirements/../../../requirements/constraints.txt
-    #   -r requirements/_test.in
 aiormq==3.3.1
     # via
     #   -c requirements/_base.txt
@@ -51,7 +47,6 @@ async-timeout==4.0.2
     # via
     #   -c requirements/_base.txt
     #   aiohttp
-    #   aioredis
 attrs==20.3.0
     # via
     #   -c requirements/_base.txt
@@ -146,8 +141,6 @@ h11==0.12.0
     # via
     #   -c requirements/_base.txt
     #   httpcore
-hiredis==2.0.0
-    # via aioredis
 httpcore==0.14.4
     # via
     #   -c requirements/_base.txt

services/director-v2/tests/conftest.py

@@ -33,7 +33,6 @@
     "pytest_simcore.postgres_service",
     "pytest_simcore.pydantic_models",
     "pytest_simcore.rabbit_service",
-    "pytest_simcore.redis_service",
     "pytest_simcore.repository_paths",
     "pytest_simcore.schemas",
     "pytest_simcore.simcore_dask_service",

services/director-v2/tests/integration/01/test_computation_api.py

@@ -24,7 +24,6 @@
 from models_library.projects_pipeline import PipelineDetails
 from models_library.projects_state import RunningState
 from settings_library.rabbit import RabbitSettings
-from settings_library.redis import RedisSettings
 from shared_comp_utils import (
     COMPUTATION_URL,
     assert_and_wait_for_pipeline_status,
@@ -44,7 +43,6 @@
     "migration",
     "postgres",
     "rabbit",
-    "redis",
     "storage",
 ]
 pytest_simcore_ops_services_selection = ["minio", "adminer"]
@@ -81,7 +79,6 @@ def minimal_configuration(
     jupyter_service: Dict[str, str],
     dask_scheduler_service: str,
     dask_sidecar_service: None,
-    redis_service: RedisSettings,
     postgres_db: sa.engine.Engine,
     postgres_host_config: Dict[str, str],
     rabbit_service: RabbitSettings,

services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py

@@ -50,7 +50,6 @@
 from pytest_mock.plugin import MockerFixture
 from pytest_simcore.helpers.utils_docker import get_localhost_ip
 from settings_library.rabbit import RabbitSettings
-from settings_library.redis import RedisSettings
 from shared_comp_utils import (
     assert_and_wait_for_pipeline_status,
     assert_computation_task_out_obj,
@@ -96,7 +95,6 @@
     "migration",
     "postgres",
     "rabbit",
-    "redis",
     "storage",
 ]
 
@@ -131,7 +129,6 @@ def minimal_configuration(  # pylint:disable=too-many-arguments
     sleeper_service: Dict,
     dy_static_file_server_dynamic_sidecar_service: Dict,
     dy_static_file_server_dynamic_sidecar_compose_spec_service: Dict,
-    redis_service: RedisSettings,
     postgres_db: sa.engine.Engine,
     postgres_host_config: Dict[str, str],
     rabbit_service: RabbitSettings,

services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py

@@ -17,7 +17,6 @@
 from pytest_mock.plugin import MockerFixture
 from pytest_simcore.helpers.utils_docker import get_localhost_ip
 from settings_library.rabbit import RabbitSettings
-from settings_library.redis import RedisSettings
 from simcore_sdk.node_ports_common import config as node_ports_config
 from simcore_service_director_v2.core.application import init_app
 from simcore_service_director_v2.core.settings import AppSettings
@@ -43,7 +42,6 @@
     "migration",
     "postgres",
     "rabbit",
-    "redis",
     "storage",
 ]
 
@@ -57,7 +55,6 @@ def minimal_configuration(
     dy_static_file_server_service: Dict,
     dy_static_file_server_dynamic_sidecar_service: Dict,
     dy_static_file_server_dynamic_sidecar_compose_spec_service: Dict,
-    redis_service: RedisSettings,
     postgres_db: sa.engine.Engine,
     postgres_host_config: Dict[str, str],
     rabbit_service: RabbitSettings,

services/web/server/requirements/_base.in

@@ -36,8 +36,6 @@ aiohttp_security
 aiohttp_session[secure]
 aiohttp-swagger[performance]
 aiopg[sa] # db
-aioredis # redis
-aioredlock # redis
 aiosmtplib # email
 asyncpg # db
 change_case
@@ -54,5 +52,6 @@ passlib
 pint # units
 pydantic[email] # models
 python-magic # excel
+redis
 semantic_version
 tenacity