Set semaphore to 2 for "request_task" API

Tested working fine with a fleet of 75k cores 10k machines,
Waitress configured with threads=4 and connection_limit=100
HW usage: RAM 72%, primary Waitress instance CPU 85-95%

This is a good compromise for threads "privileged" for "request_task" API,
and threads free for any other APIs, while keeping Waitress threads=4.

Raising the number of Waitress threads uses more RAM and can
requires lowering the MongoDB cache size to avoid the OOM killer.

see #2003

Drop old mongodb requirement.

server/fishtest/rundb.py

@@ -678,8 +678,10 @@ def update_workers_cores(self, run):
         run["workers"], run["cores"] = workers, cores
 
     # Limit concurrent request_task
+    # The semaphore must be initialized with a value
+    # less than the number of Waitress threads.
     task_lock = threading.Lock()
-    task_semaphore = threading.Semaphore(4)
+    task_semaphore = threading.Semaphore(2)
 
     task_time = 0
     task_runs = None

server/production.ini

@@ -31,7 +31,6 @@ trusted_proxy_count = 1
 trusted_proxy_headers = x-forwarded-for x-forwarded-host x-forwarded-proto x-forwarded-port
 clear_untrusted_proxy_headers = yes
 
-# Match connection limit with max number of workers
 connection_limit = 100
 threads = 4
 

server/setup.py

@@ -8,7 +8,7 @@
     "pyramid_debugtoolbar",
     "pyramid_mako",
     "waitress",
-    "pymongo > 4",
+    "pymongo",
     "numpy",
     "scipy",
     "requests",