From 7cfc6c5bc32ffc462ba3d63e4d4105afb8ab15c4 Mon Sep 17 00:00:00 2001 From: ummer-shell <124354705+ummer-shell@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:02:41 +0100 Subject: [PATCH] Batch API updates (#808) * API FIX: Ensure threadpool workers parsed as integers Signed-off-by: Ummer Taahir * API FIX: Add batch base query params rather than using normal base Signed-off-by: Ummer Taahir * Update mkdocstrings versions Signed-off-by: Ummer Taahir --------- Signed-off-by: Ummer Taahir --- environment.yml | 4 ++-- src/api/v1/batch.py | 11 +++++++---- src/api/v1/common.py | 3 +++ src/api/v1/models.py | 10 ++++++++++ 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/environment.yml b/environment.yml index aad477b70..a382d33e7 100644 --- a/environment.yml +++ b/environment.yml @@ -47,8 +47,8 @@ dependencies: - openai>=1.13.3,<2.0.0 - mkdocs-material==9.5.20 - mkdocs-material-extensions==1.3.1 - - mkdocstrings==0.22.0 - - mkdocstrings-python==1.4.0 + - mkdocstrings==0.25.0 + - mkdocstrings-python==1.10.8 - mkdocs-macros-plugin==1.0.1 - pygments==2.16.1 - pymdown-extensions==10.8.1 diff --git a/src/api/v1/batch.py b/src/api/v1/batch.py index 861b60986..4b17c772e 100755 --- a/src/api/v1/batch.py +++ b/src/api/v1/batch.py @@ -18,7 +18,7 @@ from src.sdk.python.rtdip_sdk.queries.time_series import batch from src.api.v1.models import ( - BaseQueryParams, + BatchBaseQueryParams, BaseHeaders, BatchBodyParams, BatchResponse, @@ -128,8 +128,11 @@ async def batch_events_get( # Parse requests into dicts required by sdk parsed_requests = parse_batch_requests(batch_query_parameters.requests) - # Obtain max workers from environment var, otherwise default to one less than cpu count - max_workers = os.environ.get("BATCH_THREADPOOL_WORKERS", os.cpu_count() - 1) + # Obtain max workers from environment var, otherwise default to 10 + max_workers = os.environ.get("BATCH_THREADPOOL_WORKERS", 10) + + # ensure max_workers is an integer + max_workers = int(max_workers) # Request the data for each concurrently with threadpool with ThreadPoolExecutor(max_workers=max_workers) as executor: @@ -172,7 +175,7 @@ async def batch_events_get( }, ) async def batch_post( - base_query_parameters: BaseQueryParams = Depends(), + base_query_parameters: BatchBaseQueryParams = Depends(), batch_query_parameters: BatchBodyParams = Body(default=...), base_headers: BaseHeaders = Depends(), limit_offset_query_parameters: LimitOffsetQueryParams = Depends(), diff --git a/src/api/v1/common.py b/src/api/v1/common.py index 04e657a5b..42b27e487 100644 --- a/src/api/v1/common.py +++ b/src/api/v1/common.py @@ -285,6 +285,9 @@ def lookup_before_get( # make default workers 3 as within one query typically will request from only a few tables at once max_workers = os.environ.get("LOOKUP_THREADPOOL_WORKERS", 3) + # ensure max_workers is an integer + max_workers = int(max_workers) + # run function with each parameters concurrently results = batch.get(connection, request_list, threadpool_max_workers=max_workers) diff --git a/src/api/v1/models.py b/src/api/v1/models.py index 34461f48b..d07fc9b4c 100644 --- a/src/api/v1/models.py +++ b/src/api/v1/models.py @@ -253,6 +253,16 @@ def __init__( self.authorization = authorization +class BatchBaseQueryParams: + def __init__( + self, + region: str = Query(..., description="Region"), + authorization: str = Depends(oauth2_scheme), + ): + self.region = region + self.authorization = authorization + + class MetadataQueryParams: def __init__( self,