From 5e5170fc518d6c307df49db2978bfeb1c853a7d7 Mon Sep 17 00:00:00 2001 From: Martastain Date: Fri, 9 Aug 2024 23:10:44 +0200 Subject: [PATCH 1/4] feat: add available db connections to system metrics --- ayon_server/metrics/system.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ayon_server/metrics/system.py b/ayon_server/metrics/system.py index 5b3f0b00..7971a735 100644 --- a/ayon_server/metrics/system.py +++ b/ayon_server/metrics/system.py @@ -37,6 +37,11 @@ class SystemMetricsData(OPModel): runtime_seconds: float = Field(0, title="Runtime", example=123456) db_size_shared: int = Field(0, title="Shared database size", example=123456) db_size_total: int = Field(0, title="Total database size", example=123456) + db_available_connections: int = Field( + 0, + title="Available database connections", + example=123456, + ) redis_size_total: int = Field(0, title="Total redis size", example=123456) storage_utilization_total: int = Field( 0, @@ -50,7 +55,7 @@ def __init__(self): self.boot_time = psutil.boot_time() self.run_time = time.time() - @aiocache.cached(ttl=60) + @aiocache.cached(ttl=120) async def get_db_sizes(self) -> list[Metric]: result = [] query = """ @@ -93,6 +98,7 @@ async def status(self) -> list[Metric]: Metric("uptime_seconds", time.time() - self.boot_time), Metric("runtime_seconds", time.time() - self.run_time), Metric("redis_size_total", redis_size), + Metric("db_available_connections", Postgres.get_available_connections()), ] async def render_prometheus(self) -> str: @@ -107,7 +113,7 @@ async def render_prometheus(self) -> str: result += metric.render_prometheus() return result - @aiocache.cached(ttl=60) + @aiocache.cached(ttl=120) async def get_upload_sizes(self) -> list[Metric]: result: list[Metric] = [] From d72d6e1552195f95daaec9682bb6fa91129e5d05 Mon Sep 17 00:00:00 2001 From: Martastain Date: Fri, 9 Aug 2024 23:11:10 +0200 Subject: [PATCH 2/4] feat: make db pool size configurable, increase default value --- ayon_server/config.py | 6 ++++++ ayon_server/lib/postgres.py | 9 ++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ayon_server/config.py b/ayon_server/config.py index 3f906108..26ebd775 100644 --- a/ayon_server/config.py +++ b/ayon_server/config.py @@ -91,6 +91,12 @@ class AyonConfig(BaseModel): example="postgres://user:password123@postgres.example.com:5432/ayon", ) + postgres_pool_size: int = Field( + 64, + description="Postgres connection pool size", + example=64, + ) + session_ttl: int = Field( default=24 * 3600, description="Session lifetime in seconds", diff --git a/ayon_server/lib/postgres.py b/ayon_server/lib/postgres.py index 1a51d558..8ef64930 100644 --- a/ayon_server/lib/postgres.py +++ b/ayon_server/lib/postgres.py @@ -82,12 +82,7 @@ def get_available_connections(cls) -> int: @classmethod async def connect(cls) -> None: - """Create a PostgreSQL connection pool. - - By default, we use 24 as max_size, since default maximum - connection value of postgres is 100 so, 25 is perfect for - 4 workers and a small reserve. - """ + """Create a PostgreSQL connection pool.""" if cls.shutting_down: print("Unable to connect to Postgres while shutting down.") @@ -95,7 +90,7 @@ async def connect(cls) -> None: cls.pool = await asyncpg.create_pool( ayonconfig.postgres_url, min_size=10, - max_size=24, + max_size=ayonconfig.postgres_pool_size, max_inactive_connection_lifetime=20, init=cls.init_connection, ) From cc1193e63b95897f3eaa3eb475bb3364264d452c Mon Sep 17 00:00:00 2001 From: Martastain Date: Tue, 13 Aug 2024 22:20:21 +0200 Subject: [PATCH 3/4] feat: configurable db pool acquire timeout --- ayon_server/config.py | 8 +++++++- ayon_server/lib/postgres.py | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ayon_server/config.py b/ayon_server/config.py index 26ebd775..0b166336 100644 --- a/ayon_server/config.py +++ b/ayon_server/config.py @@ -97,6 +97,12 @@ class AyonConfig(BaseModel): example=64, ) + postgres_pool_timeout: int = Field( + 20, + description="Postgres connection pool timeout", + example=20, + ) + session_ttl: int = Field( default=24 * 3600, description="Session lifetime in seconds", @@ -178,7 +184,7 @@ class AyonConfig(BaseModel): http_timeout: int = Field( default=120, - description="Timeout for HTTP requests the server uses " + description="The default timeout for HTTP requests the server uses " "to connect to external services", ) diff --git a/ayon_server/lib/postgres.py b/ayon_server/lib/postgres.py index 8ef64930..ac7ebe14 100644 --- a/ayon_server/lib/postgres.py +++ b/ayon_server/lib/postgres.py @@ -23,7 +23,6 @@ class Postgres: Provides an interface for interacting with a Postgres database. """ - default_acquire_timeout: int = 10 shutting_down: bool = False pool: asyncpg.pool.Pool | None = None # type: ignore @@ -42,7 +41,7 @@ async def acquire( raise ConnectionError("Connection pool is not initialized.") if timeout is None: - timeout = cls.default_acquire_timeout + timeout = ayonconfig.postgres_pool_timeout try: connection_proxy = await cls.pool.acquire(timeout=timeout) From fd6f616350ec6628c7b10e3f11d73da82a3f2e07 Mon Sep 17 00:00:00 2001 From: Martastain Date: Tue, 13 Aug 2024 22:21:22 +0200 Subject: [PATCH 4/4] fix: db pool size prometheus metrics --- ayon_server/helpers/deploy_project.py | 2 +- ayon_server/metrics/system.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ayon_server/helpers/deploy_project.py b/ayon_server/helpers/deploy_project.py index a91399a9..a763e764 100644 --- a/ayon_server/helpers/deploy_project.py +++ b/ayon_server/helpers/deploy_project.py @@ -136,7 +136,7 @@ async def create_project_from_anatomy( await assign_default_users_to_project(project.name, conn) end_time = time.monotonic() - logging.info(f"Deployed project {project.name} in {end_time - start_time:.2f}s") + logging.debug(f"Deployed project {project.name} in {end_time - start_time:.2f}s") await dispatch_event( "entity.project.created", diff --git a/ayon_server/metrics/system.py b/ayon_server/metrics/system.py index 7971a735..5f151865 100644 --- a/ayon_server/metrics/system.py +++ b/ayon_server/metrics/system.py @@ -98,7 +98,6 @@ async def status(self) -> list[Metric]: Metric("uptime_seconds", time.time() - self.boot_time), Metric("runtime_seconds", time.time() - self.run_time), Metric("redis_size_total", redis_size), - Metric("db_available_connections", Postgres.get_available_connections()), ] async def render_prometheus(self) -> str: @@ -106,11 +105,18 @@ async def render_prometheus(self) -> str: for metric in await self.status(): result += metric.render_prometheus() - for metric in await self.get_db_sizes(): + for metric in await self.get_upload_sizes(): result += metric.render_prometheus() - for metric in await self.get_upload_sizes(): + for metric in await self.get_db_sizes(): result += metric.render_prometheus() + + # ~realtime data + db_avail = Metric( + "db_available_connections", Postgres.get_available_connections() + ) + result += db_avail.render_prometheus() + return result @aiocache.cached(ttl=120)