From 31a3923399baca2c3e307af593a285df6d05e60e Mon Sep 17 00:00:00 2001
From: garciam <garciam@predictia.es>
Date: Wed, 3 Jul 2024 13:51:53 +0200
Subject: [PATCH] improve load tests

---
 tests/scripts/make_load_test_csv.py |  2 +-
 tests/scripts/run_load_test.py      | 28 +++++++++++++++-------------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/tests/scripts/make_load_test_csv.py b/tests/scripts/make_load_test_csv.py
index 3b80ed3..00dd840 100644
--- a/tests/scripts/make_load_test_csv.py
+++ b/tests/scripts/make_load_test_csv.py
@@ -58,7 +58,7 @@ def main():
 
 
 def get_load_test_requests_dataset(collection_id):
-    constraints_dir = Path("/home/garciam/git/copds/cads-forms-json")
+    constraints_dir = Path("../../../cads-forms-json")
     requests_per_type = 33
 
     with Path(constraints_dir, collection_id, "constraints.json").open("r") as cfile:
diff --git a/tests/scripts/run_load_test.py b/tests/scripts/run_load_test.py
index 7e08203..6408e98 100644
--- a/tests/scripts/run_load_test.py
+++ b/tests/scripts/run_load_test.py
@@ -3,29 +3,34 @@
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 
-logging.getLogger("root").setLevel("DEBUG")
+formatter = logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s")
+logger = logging.getLogger(__name__)
+logger.setLevel("INFO")
+handler = logging.StreamHandler()
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+logger.propagate = False
 
 
 def run_request(client, collection_id, request):
-    result = client.retrieve(collection_id, **request)
-    return result
+    try:
+        client.retrieve(collection_id, **request)
+    except Exception as e:
+        logger.error(f"Exception {e} captured for request {collection_id=} {request=}")
 
 
 def main():
     import cads_api_client
 
-    url = "https://cds-dev-bopen.copernicus-climate.eu/api/"
-    ifile = Path(
-        "load_test_requests_insitu-observations-woudc-ozone-total-column-and-profiles.jsonl"
-    )
-    client = cads_api_client.ApiClient(url=url)
-    test_N = 1
+    ifile = Path("load_test_requests.jsonl")
+    client = cads_api_client.ApiClient()
+    test_N = 500
     futures = []
     with ThreadPoolExecutor(max_workers=10) as executor:
         with ifile.open("r") as ifileobj:
             for i, line in enumerate(ifileobj, start=1):
                 collection_id, request_params = json.loads(line)
-                print(f"Submitting request for {collection_id}, {request_params}")
+                logger.info(f"Submitting request for {collection_id}, {request_params}")
                 future = executor.submit(
                     run_request, client, collection_id, request_params
                 )
@@ -33,9 +38,6 @@ def main():
                 if i == test_N:
                     break
 
-            for f in futures:
-                print(f.result())
-
 
 if __name__ == "__main__":
     main()