Merge pull request #20 from ecmwf-projects/improve-load-tests

improve load tests

tests/scripts/make_load_test_csv.py

@@ -58,7 +58,7 @@ def main():
 
 
 def get_load_test_requests_dataset(collection_id):
-    constraints_dir = Path("/home/garciam/git/copds/cads-forms-json")
+    constraints_dir = Path("../../../cads-forms-json")
     requests_per_type = 33
 
     with Path(constraints_dir, collection_id, "constraints.json").open("r") as cfile:

tests/scripts/run_load_test.py

@@ -3,39 +3,41 @@
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 
-logging.getLogger("root").setLevel("DEBUG")
+formatter = logging.Formatter("%(asctime)s — %(name)s — %(levelname)s — %(message)s")
+logger = logging.getLogger(__name__)
+logger.setLevel("INFO")
+handler = logging.StreamHandler()
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+logger.propagate = False
 
 
 def run_request(client, collection_id, request):
-    result = client.retrieve(collection_id, **request)
-    return result
+    try:
+        client.retrieve(collection_id, **request)
+    except Exception as e:
+        logger.error(f"Exception {e} captured for request {collection_id=} {request=}")
 
 
 def main():
     import cads_api_client
 
-    url = "https://cds-dev-bopen.copernicus-climate.eu/api/"
-    ifile = Path(
-        "load_test_requests_insitu-observations-woudc-ozone-total-column-and-profiles.jsonl"
-    )
-    client = cads_api_client.ApiClient(url=url)
-    test_N = 1
+    ifile = Path("load_test_requests.jsonl")
+    client = cads_api_client.ApiClient()
+    test_N = 500
     futures = []
     with ThreadPoolExecutor(max_workers=10) as executor:
         with ifile.open("r") as ifileobj:
             for i, line in enumerate(ifileobj, start=1):
                 collection_id, request_params = json.loads(line)
-                print(f"Submitting request for {collection_id}, {request_params}")
+                logger.info(f"Submitting request for {collection_id}, {request_params}")
                 future = executor.submit(
                     run_request, client, collection_id, request_params
                 )
                 futures.append(future)
                 if i == test_N:
                     break
 
-            for f in futures:
-                print(f.result())
-
 
 if __name__ == "__main__":
     main()