Merge pull request #1519 from chaoss/sean-patch-aaf

Schema, configuration, facade worker (commits) and housekeeper updates
chaoss · Nov 17, 2021 · a59c0fb · a59c0fb
2 parents 21d094d + 031e981
commit a59c0fb
Show file tree

Hide file tree

Showing 9 changed files with 42 additions and 28 deletions.
diff --git a/augur/config.py b/augur/config.py
@@ -411,8 +411,8 @@
             "cache_expire": "3600",
             "host": "0.0.0.0",
             "port": main_port,
-            "workers": 12,
-            "timeout": 60000
+            "workers": 6,
+            "timeout": 6000
         },
         "Frontend": {
             "host": "0.0.0.0",

diff --git a/augur/housekeeper.py b/augur/housekeeper.py
@@ -64,7 +64,6 @@ def schedule_updates(self):
         logger.info("Scheduling update processes")
         for job in self.jobs:
             process = Process(target=self.updater_process, name=job["model"], args=(self.broker_host, self.broker_port, self.broker, job, (self.augur_logging.housekeeper_job_config, self.augur_logging.get_config())))
-            logger.debug(f'starting process {process}')
             self._processes.append(process)
             process.start()
 
@@ -107,7 +106,6 @@ def updater_process(broker_host, broker_port, broker, job, logging_config):
                         job['model'], job['given'][0]))
 
                     if job['given'][0] == 'git_url' or job['given'][0] == 'github_url':
-                        logger.debug(f'job array is: {job}')
                         for repo in job['repos']:
                             if job['given'][0] == 'github_url' and 'github.com' not in repo['repo_git']:
                                 continue
@@ -132,7 +130,6 @@ def updater_process(broker_host, broker_port, broker, job, logging_config):
                             time.sleep(15)
 
                     elif job['given'][0] == 'repo_group':
-                        logger.debug(f'job array is: {job}')
                         task = {
                                 "job_type": job['job_type'] if 'job_type' in job else 'MAINTAIN', 
                                 "models": [job['model']], 
@@ -143,14 +140,10 @@ def updater_process(broker_host, broker_port, broker, job, logging_config):
                             }
                         try:
                             requests.post('http://{}:{}/api/unstable/task'.format(
-                                broker_host,broker_port), json=task, timeout=30)
+                                broker_host,broker_port), json=task, timeout=10)
                         except Exception as e:
                             logger.error("Error encountered: {}".format(e))
 
-                        logger.debug(task)
-
-                        time.sleep(15)
-
                     logger.info("Housekeeper finished sending {} tasks to the broker for it to distribute to your worker(s)".format(len(job['repos'])))
                     time.sleep(job['delay'])
 
@@ -247,7 +240,6 @@ def prep_jobs(self):
                         {}
                         group by repo.repo_id ORDER BY commit_count {}
                     """.format(where_condition, job['order']))
-                logger.debug(f'repo url sql is: {repo_url_sql}. \n\n \n \n  where condition is {where_condition} \n \n \n \n and the where_and condition is {where_and}\n \n \n ')
 
                 reorganized_repos = pd.read_sql(repo_url_sql, self.db, params={})
                 if len(reorganized_repos) == 0:
@@ -310,8 +302,6 @@ def prep_jobs(self):
 
                 if finishing_task:
                     reorganized_repos[0]['focused_task'] = 1
-
-                #logger.debug(f'reorganized repos == {reorganized_repos}.')
 
                 job['repos'] = reorganized_repos
 

diff --git a/augur/routes/broker.py b/augur/routes/broker.py
@@ -75,7 +75,7 @@ def task():
         for given_component in list(task['given'].keys()):
             given.append(given_component)
         model = task['models'][0]
-        logger.info("Broker received a new user task ... checking for compatible workers for given: " + str(given) + " and model(s): " + str(model) + "\n")
+        logger.info("Broker recieved a new user task ... checking for compatible workers for given: " + str(given) + " and model(s): " + str(model) + "\n")
 
         logger.debug("Broker's list of all workers: {}\n".format(server.broker._getvalue().keys()))
 
@@ -128,7 +128,7 @@ def worker():
             and telling the broker to add this worker to the set it maintains
         """
         worker = request.json
-        logger.info("Received HELLO message from worker {} listening on: https://localhost:{}\
+        logger.info("Recieved HELLO message from worker {} listening on: https://localhost:{}\
                     ".format(worker['id'], worker['id'].split('.')[2]))
         if worker['id'] not in server.broker:
             server.broker[worker['id']] = server.manager.dict()
@@ -162,7 +162,7 @@ def worker():
     def sync_queue():
         task = request.json
         worker = task['worker_id']
-        logger.info("Message received that worker {} completed task: {}\n".format(worker,task))
+        logger.info("Message recieved that worker {} completed task: {}\n".format(worker,task))
         try:
             models = server.broker[worker]['models']
             givens = server.broker[worker]['given']
@@ -202,7 +202,7 @@ def get_status():
     @server.app.route('/{}/workers/remove'.format(server.api_version), methods=['POST'])
     def remove_worker():
         worker = request.json
-        logger.info("Received a message to disconnect worker: {}\n".format(worker))
+        logger.info("Recieved a message to disconnect worker: {}\n".format(worker))
         server.broker[worker['id']]['status'] = 'Disconnected'
         return Response(response=worker,
                         status=200,
@@ -212,7 +212,7 @@ def remove_worker():
     def task_error():
         task = request.json
         worker_id = task['worker_id']
-        # logger.error("Received a message that {} ran into an error on task: {}\n".format(worker_id, task))
+        # logger.error("Recieved a message that {} ran into an error on task: {}\n".format(worker_id, task))
         if worker_id in server.broker:
             if server.broker[worker_id]['status'] != 'Disconnected':
                 logger.error("{} ran into error while completing task: {}\n".format(worker_id, task))
@@ -221,4 +221,4 @@ def task_error():
             logger.error("A previous instance of {} ran into error while completing task: {}\n".format(worker_id, task))
         return Response(response=request.json,
                         status=200,
-                        mimetype="application/json")
+                        mimetype="application/json")
diff --git a/metadata.py b/metadata.py
@@ -5,8 +5,8 @@
 
 __short_description__ = "Python 3 package for free/libre and open-source software community metrics & data collection"
 
-__version__ = "0.21.8"
-__release__ = "v0.21.8"
+__version__ = "0.21.9"
+__release__ = "v0.21.9"
 
 __license__ = "MIT"
 __copyright__ = "CHAOSS & Augurlabs 2021"
diff --git a/schema/create_schema.sql b/schema/create_schema.sql
@@ -7,6 +7,7 @@
 \i schema/generate/80-schema_update_82.sql
 \i schema/generate/81-schema_update_83.sql
 \i schema/generate/82-schema_update_84.sql
+\i schema/generate/83-schema_update_85.sql
 
 
 

diff --git a/schema/generate/05-seed_data.sql b/schema/generate/05-seed_data.sql
@@ -1083,3 +1083,5 @@ INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_p
 UPDATE "augur_data"."repo" set repo_name = NULL, repo_path = NULL, repo_status = 'New'; 
 
 
+INSERT INTO "augur_operations"."worker_history" ("history_id", "repo_id", "worker", "job_model", "oauth_id", "timestamp", "status", "total_results") VALUES (1, 1, 'workers.repo_info_worker.50723', 'repo_info', 0, '2021-10-17 12:05:22', 'Success', 1);
+
diff --git a/schema/generate/82-schema_update_84.sql b/schema/generate/82-schema_update_84.sql
@@ -1,8 +1,5 @@
 BEGIN; 
 ALTER TABLE "augur_data"."repo" 
-  ALTER COLUMN "tool_source" SET DEFAULT CLI and API,
-  ALTER COLUMN "tool_version" SET DEFAULT See release notes,
-  ALTER COLUMN "data_source" SET DEFAULT Data Load,
   ALTER COLUMN "data_collection_date" SET DEFAULT CURRENT_TIMESTAMP;
 update "augur_operations"."augur_settings" set value = 84 where setting = 'augur_data_version'; 
 

diff --git a/schema/generate/83-schema_update_85.sql b/schema/generate/83-schema_update_85.sql
@@ -0,0 +1,24 @@
+BEGIN; 
+ALTER TABLE "augur_data"."pull_request_review_message_ref" 
+  DROP CONSTRAINT "fk_pull_request_review_message_ref_message_1",
+  DROP CONSTRAINT "fk_pull_request_review_message_ref_pull_request_reviews_1",
+  ADD CONSTRAINT "fk_pull_request_review_message_ref_message_1" FOREIGN KEY ("msg_id") REFERENCES "augur_data"."message" ("msg_id") ON DELETE RESTRICT ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  ADD CONSTRAINT "fk_pull_request_review_message_ref_pull_request_reviews_1" FOREIGN KEY ("pr_review_id") REFERENCES "augur_data"."pull_request_reviews" ("pr_review_id") ON DELETE RESTRICT ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  ADD CONSTRAINT "fk_review_repo" FOREIGN KEY ("repo_id") REFERENCES "augur_data"."repo" ("repo_id") ON DELETE RESTRICT ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED;
+
+ALTER TABLE "augur_data"."pull_request_message_ref" 
+  DROP CONSTRAINT "fk_pull_request_message_ref_message_1",
+  DROP CONSTRAINT "fk_pull_request_message_ref_pull_requests_1",
+  ADD CONSTRAINT "fk_pull_request_message_ref_message_1" FOREIGN KEY ("msg_id") REFERENCES "augur_data"."message" ("msg_id") ON DELETE RESTRICT ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  ADD CONSTRAINT "fk_pull_request_message_ref_pull_requests_1" FOREIGN KEY ("pull_request_id") REFERENCES "augur_data"."pull_requests" ("pull_request_id") ON DELETE RESTRICT ON UPDATE CASCADE DEFERRABLE INITIALLY DEFERRED,
+  ADD CONSTRAINT "fk_pr_repo" FOREIGN KEY ("repo_id") REFERENCES "augur_data"."repo" ("repo_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+ALTER TABLE "augur_data"."issue_message_ref" 
+  ALTER CONSTRAINT "fk_issue_message_ref_issues_1" DEFERRABLE INITIALLY DEFERRED,
+  ALTER CONSTRAINT "fk_issue_message_ref_message_1" DEFERRABLE INITIALLY DEFERRED,
+  ALTER CONSTRAINT "fk_repo_id_fk1" DEFERRABLE INITIALLY DEFERRED;
+
+update "augur_operations"."augur_settings" set value = 85 where setting = 'augur_data_version'; 
+
+
+COMMIT; 
diff --git a/workers/facade_worker/facade_worker/facade05repofetch.py b/workers/facade_worker/facade_worker/facade05repofetch.py
@@ -307,7 +307,7 @@ def git_repo_updates(cfg):
                 firstpull = ("git -C %s%s/%s%s pull"
                     % (cfg.repo_base_directory,row[1],row[4],row[3]))
 
-                return_code_remote = subprocess.Popen([firstpull],stdout=subprocess.PIPE,shell=True).wait()
+                return_code_remote = subprocess.Popen([firstpull],shell=True).wait()
 
                 cfg.log_activity('Verbose', 'Got to here. 1.')
 
@@ -336,7 +336,7 @@ def git_repo_updates(cfg):
 
                     cfg.log_activity('Verbose', f'get remote default command is: \n \n {getremotedefault} \n \n ')
 
-                    return_code_remote_default_again = subprocess.Popen([getremotedefault],stdout=subprocess.PIPE,shell=True).wait()
+                    return_code_remote_default_again = subprocess.Popen([getremotedefault],shell=True).wait()
 
                     if return_code_remote_default_again == 0: 
                         cfg.log_activity('Verbose', "local checkout worked.")
@@ -409,15 +409,15 @@ def git_repo_updates(cfg):
                     cmd_checkout_default =  (f"git -C %s%s/%s%s checkout {remote_default}" 
                     % (cfg.repo_base_directory,row[1],row[4],row[3]))
 
-                    cmd_checkout_default_wait = subprocess.Popen([cmd_checkout_default],stdout=subprocess.PIPE,shell=True).wait()
+                    cmd_checkout_default_wait = subprocess.Popen([cmd_checkout_default],shell=True).wait()
 
                     cmdpull2 = ("git -C %s%s/%s%s pull"
                         % (cfg.repo_base_directory,row[1],row[4],row[3]))
 
                     cmd_reset = ("git -C %s%s/%s%s reset --hard origin"
                         % (cfg.repo_base_directory,row[1],row[4],row[3]))
 
-                    cmd_reset_wait = subprocess.Popen([cmd_reset],stdout=subprocess.PIPE,shell=True).wait()
+                    cmd_reset_wait = subprocess.Popen([cmd_reset],shell=True).wait()
 
                     cmd_clean = ("git -C %s%s/%s%s clean -df"
                         % (cfg.repo_base_directory,row[1],row[4],row[3]))
Original file line number	Diff line number	Diff line change
Expand Up		@@ -1083,3 +1083,5 @@ INSERT INTO "augur_data"."repo" ("repo_id", "repo_group_id", "repo_git", "repo_p
		UPDATE "augur_data"."repo" set repo_name = NULL, repo_path = NULL, repo_status = 'New';


		INSERT INTO "augur_operations"."worker_history" ("history_id", "repo_id", "worker", "job_model", "oauth_id", "timestamp", "status", "total_results") VALUES (1, 1, 'workers.repo_info_worker.50723', 'repo_info', 0, '2021-10-17 12:05:22', 'Success', 1);