Merge pull request #1522 from chaoss/sean-patch-aaf

Updated Housekeeper sleep parameters to limit initial "Augur Surge"
chaoss · Nov 18, 2021 · 31021d5 · 31021d5
2 parents a59c0fb + 778d5ed
commit 31021d5
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 2 deletions.
diff --git a/augur/housekeeper.py b/augur/housekeeper.py
@@ -96,7 +96,7 @@ def updater_process(broker_host, broker_port, broker, job, logging_config):
                     for worker in list(broker._getvalue().keys()):
                         if job['model'] in broker[worker]['models'] and job['given'] in broker[worker]['given']:
                             compatible_worker_found = True
-                    time.sleep(3)
+                    time.sleep(120)
                     continue
 
                 logger.info("Housekeeper recognized that the broker has a worker that " + 
@@ -127,7 +127,7 @@ def updater_process(broker_host, broker_port, broker, job, logging_config):
 
                             logger.debug(task)
 
-                            time.sleep(15)
+                            time.sleep(120)
 
                     elif job['given'][0] == 'repo_group':
                         task = {
@@ -141,6 +141,7 @@ def updater_process(broker_host, broker_port, broker, job, logging_config):
                         try:
                             requests.post('http://{}:{}/api/unstable/task'.format(
                                 broker_host,broker_port), json=task, timeout=10)
+                            time.sleep(120)
                         except Exception as e:
                             logger.error("Error encountered: {}".format(e))
 

diff --git a/workers/worker_git_integration.py b/workers/worker_git_integration.py
@@ -2,6 +2,7 @@
 from numpy.lib.utils import source
 from workers.worker_base import *
 import sqlalchemy as s
+import time 
 
 #This is a worker base subclass that adds the ability to query github/gitlab with the api key
 class WorkerGitInterfaceable(Worker):
@@ -970,6 +971,7 @@ def update_gh_rate_limit(self, response, bad_credentials=False, temporarily_disa
             except:
                 self.oauths[0]['rate_limit'] -= 1
                 self.logger.info("Headers did not work, had to decrement")
+                time.sleep(30)
         self.logger.info(
             f"Updated rate limit, you have: {self.oauths[0]['rate_limit']} requests remaining."
         )
@@ -1405,10 +1407,15 @@ def paginate_endpoint(
                             f"{url.format(page_number)}\n"
                         )
                         break
+                    if "You have exceeded a secondary rate limit. Please wait a few minutes before you try again" in page_data['message']:
+                        num_attempts -=1
+                        self.logger.info('\n\n\n\nSleeping for 100 seconds due to secondary rate limit issue.\n\n\n\n')
+                        time.sleep(100)
                     if "You have triggered an abuse detection mechanism." in page_data['message']:
                         num_attempts -= 1
                         self.update_rate_limit(response, temporarily_disable=True,platform=platform)
                     if page_data['message'] == "Bad credentials":
+                        self.logger.info("\n\n\n\n\n\n\n POSSIBLY BAD TOKEN \n\n\n\n\n\n\n")
                         self.update_rate_limit(response, bad_credentials=True, platform=platform)
                 elif type(page_data) == str:
                     self.logger.info(f"Warning! page_data was string: {page_data}\n")