diff --git a/metadata.py b/metadata.py index d529a239db..98a770e80a 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.26.5" -__release__ = "v0.26.5" +__version__ = "0.26.7" +__release__ = "v0.26.7" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, & Augurlabs 2022" diff --git a/schema/generate/100-schema_update_102.sql b/schema/generate/100-schema_update_102.sql index db316fd7db..d1a822bb19 100644 --- a/schema/generate/100-schema_update_102.sql +++ b/schema/generate/100-schema_update_102.sql @@ -218,8 +218,8 @@ CREATE MATERIALIZED VIEW "augur_data"."explorer_entry_list" AS SELECT DISTINCT r.repo_git, rg.rg_name - FROM (repo r - JOIN repo_groups rg ON ((rg.repo_group_id = r.repo_group_id))) + FROM (augur_data.repo r + JOIN augur_data.repo_groups rg ON ((rg.repo_group_id = r.repo_group_id))) ORDER BY rg.rg_name; ALTER MATERIALIZED VIEW "augur_data"."explorer_entry_list" OWNER TO "augur"; diff --git a/user-resolution-rate-check.sh b/user-resolution-rate-check.sh new file mode 100755 index 0000000000..5e64e88a1c --- /dev/null +++ b/user-resolution-rate-check.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cat logs/workers/facade_worker/facade_worker_*_collection.log | grep "Processing repo contributors for repo:" | wc -l diff --git a/workers/worker_git_integration.py b/workers/worker_git_integration.py index 038b265c54..7b20c6c68b 100644 --- a/workers/worker_git_integration.py +++ b/workers/worker_git_integration.py @@ -6,6 +6,7 @@ import sqlalchemy as s import time import math +from random import randint #This is a worker base subclass that adds the ability to query github/gitlab with the api key class WorkerGitInterfaceable(Worker): @@ -212,11 +213,14 @@ def init_oauths(self, platform='github'): for oauth in [{'oauth_id': 0, 'access_token': self.config[key_name]}] + json.loads( pd.read_sql(oauthSQL, self.helper_db, params={}).to_json(orient="records") ): + self.logger.debug('getting oauth.') if platform == 'github': self.headers = {'Authorization': 'token %s' % oauth['access_token']} + self.logger.debug('in github oauth block') elif platform == 'gitlab': self.headers = {'Authorization': 'Bearer %s' % oauth['access_token']} - response = requests.get(url=url, headers=self.headers, timeout=180) + ## Changed timeout from 180 to 12. Seems to be hanging in some workers. + response = requests.get(url=url, headers=self.headers) self.oauths.append({ 'oauth_id': oauth['oauth_id'], 'access_token': oauth['access_token'], @@ -237,10 +241,13 @@ def init_oauths(self, platform='github'): # First key to be used will be the one specified in the config (first element in # self.oauths array will always be the key in use) + ## Attempt to get this to circulate the keys more spg 6/7/2022 + availablekeys = len(self.oauths) + keytouse = randint(0,availablekeys-1) if platform == 'github': - self.headers = {'Authorization': 'token %s' % self.oauths[0]['access_token']} + self.headers = {'Authorization': 'token %s' % self.oauths[keytouse]['access_token']} elif platform == 'gitlab': - self.headers = {'Authorization': 'Bearer %s' % self.oauths[0]['access_token']} + self.headers = {'Authorization': 'Bearer %s' % self.oauths[keytouse]['access_token']} self.logger.info("OAuth initialized\n")