Merge pull request #112 from mcmod-info-mirror/no-task-queue

feat: 将同步队列换到 redis 定期处理

.github/workflows/gitlab-sync.yml

@@ -2,8 +2,8 @@
 name: sync2gitlab
 on:
   push:
-    branches:
-      - main 
+    # branches:
+    #   - main 
 jobs:
   repo-sync:
     env:

app/__init__.py

@@ -5,14 +5,15 @@
 from starlette.middleware.cors import CORSMiddleware
 from fastapi.middleware.gzip import GZipMiddleware
 from contextlib import asynccontextmanager
-
 from app.controller import controller_router
 from app.utils.loger import log
 from app.config import MCIMConfig
 from app.database.mongodb import setup_async_mongodb, init_mongodb_aioengine
 from app.database._redis import (
     init_redis_aioengine,
     close_aio_redis_engine,
+    init_sync_queue_redis_engine,
+    close_sync_queue_redis_engine,
 )
 from app.utils.response_cache import Cache
 from app.utils.response_cache import cache
@@ -26,6 +27,7 @@
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     app.state.aio_redis_engine = init_redis_aioengine()
+    init_sync_queue_redis_engine()
     await app.state.aio_redis_engine.flushall()
     app.state.aio_mongo_engine = init_mongodb_aioengine()
     await setup_async_mongodb(app.state.aio_mongo_engine)
@@ -36,6 +38,7 @@ async def lifespan(app: FastAPI):
     yield
 
     await close_aio_redis_engine()
+    await close_sync_queue_redis_engine()
 
 
 APP = FastAPI(
@@ -58,8 +61,8 @@ async def lifespan(app: FastAPI):
 # 强制同步中间件 force=True
 APP.add_middleware(ForceSyncMiddleware)
 
-# Etag 中间件
-APP.add_middleware(EtagMiddleware)
+# # Etag 中间件
+# APP.add_middleware(EtagMiddleware)
 
 # 统计 Trustable 请求
 APP.add_middleware(CountTrustableMiddleware)

app/config/redis.py

@@ -14,20 +14,18 @@ class RedisDatabaseModel(BaseModel):
     tasks_queue: int = 0  # dramatiq tasks
     info_cache: int = 1  # response_cache and static info
     rate_limit: int = 3  # rate limit
-
+    sync_queue: int = 4  # sync queue
 
 class RedisdbConfigModel(BaseModel):
     host: str = "redis"
     port: int = 6379
-    auth: bool = True
     user: Optional[str] = None
     password: Optional[str] = None
     database: RedisDatabaseModel = RedisDatabaseModel()
 
 class SyncRedisdbConfigModel(BaseModel):
     host: str = "sync_redis"
     port: int = 6379
-    auth: bool = True
     user: Optional[str] = None
     password: Optional[str] = None