Fixing Schema Script and Addressing GH API Timeout.

metadata.py

@@ -5,8 +5,8 @@
 
 __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection"
 
-__version__ = "0.26.5"
-__release__ = "v0.26.5"
+__version__ = "0.26.7"
+__release__ = "v0.26.7"
 
 __license__ = "MIT"
 __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, & Augurlabs 2022"

schema/generate/100-schema_update_102.sql

@@ -218,8 +218,8 @@ CREATE MATERIALIZED VIEW "augur_data"."explorer_entry_list"
 AS
 SELECT DISTINCT r.repo_git,
     rg.rg_name
-   FROM (repo r
-     JOIN repo_groups rg ON ((rg.repo_group_id = r.repo_group_id)))
+   FROM (augur_data.repo r
+     JOIN augur_data.repo_groups rg ON ((rg.repo_group_id = r.repo_group_id)))
   ORDER BY rg.rg_name; 
 
 ALTER MATERIALIZED VIEW "augur_data"."explorer_entry_list" OWNER TO "augur";

user-resolution-rate-check.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+cat logs/workers/facade_worker/facade_worker_*_collection.log | grep "Processing repo contributors for repo:" | wc -l

workers/worker_git_integration.py

@@ -6,6 +6,7 @@
 import sqlalchemy as s
 import time
 import math
+from random import randint
 
 #This is a worker base subclass that adds the ability to query github/gitlab with the api key
 class WorkerGitInterfaceable(Worker):
@@ -212,11 +213,14 @@ def init_oauths(self, platform='github'):
         for oauth in [{'oauth_id': 0, 'access_token': self.config[key_name]}] + json.loads(
             pd.read_sql(oauthSQL, self.helper_db, params={}).to_json(orient="records")
         ):
+            self.logger.debug('getting oauth.')
             if platform == 'github':
                 self.headers = {'Authorization': 'token %s' % oauth['access_token']}
+                self.logger.debug('in github oauth block')
             elif platform == 'gitlab':
                 self.headers = {'Authorization': 'Bearer %s' % oauth['access_token']}
-            response = requests.get(url=url, headers=self.headers, timeout=180)
+            ## Changed timeout from 180 to 12. Seems to be hanging in some workers. 
+            response = requests.get(url=url, headers=self.headers)
             self.oauths.append({
                     'oauth_id': oauth['oauth_id'],
                     'access_token': oauth['access_token'],
@@ -237,10 +241,13 @@ def init_oauths(self, platform='github'):
 
         # First key to be used will be the one specified in the config (first element in
         #   self.oauths array will always be the key in use)
+        ## Attempt to get this to circulate the keys more spg 6/7/2022
+        availablekeys = len(self.oauths)
+        keytouse = randint(0,availablekeys-1)
         if platform == 'github':
-            self.headers = {'Authorization': 'token %s' % self.oauths[0]['access_token']}
+            self.headers = {'Authorization': 'token %s' % self.oauths[keytouse]['access_token']}
         elif platform == 'gitlab':
-            self.headers = {'Authorization': 'Bearer %s' % self.oauths[0]['access_token']}
+            self.headers = {'Authorization': 'Bearer %s' % self.oauths[keytouse]['access_token']}
 
         self.logger.info("OAuth initialized\n")