From 5401594ab7d4472ad2fe7316c61f92df014a798b Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 7 Jun 2022 13:14:30 -0500 Subject: [PATCH 1/8] debugging for oauth. --- workers/worker_git_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workers/worker_git_integration.py b/workers/worker_git_integration.py index 038b265c54..f0c963a196 100644 --- a/workers/worker_git_integration.py +++ b/workers/worker_git_integration.py @@ -212,8 +212,10 @@ def init_oauths(self, platform='github'): for oauth in [{'oauth_id': 0, 'access_token': self.config[key_name]}] + json.loads( pd.read_sql(oauthSQL, self.helper_db, params={}).to_json(orient="records") ): + self.logger.debug('getting oauth.') if platform == 'github': self.headers = {'Authorization': 'token %s' % oauth['access_token']} + self.logger.debug('in github oauth block') elif platform == 'gitlab': self.headers = {'Authorization': 'Bearer %s' % oauth['access_token']} response = requests.get(url=url, headers=self.headers, timeout=180) From 1469f8d6250121bf4c92c57b1f7d86339bffa751 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 7 Jun 2022 13:22:41 -0500 Subject: [PATCH 2/8] Attempt to rotate api keys more. --- workers/worker_git_integration.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/workers/worker_git_integration.py b/workers/worker_git_integration.py index f0c963a196..8399822bf5 100644 --- a/workers/worker_git_integration.py +++ b/workers/worker_git_integration.py @@ -6,6 +6,7 @@ import sqlalchemy as s import time import math +from random import randint #This is a worker base subclass that adds the ability to query github/gitlab with the api key class WorkerGitInterfaceable(Worker): @@ -239,10 +240,13 @@ def init_oauths(self, platform='github'): # First key to be used will be the one specified in the config (first element in # self.oauths array will always be the key in use) + ## Attempt to get this to circulate the keys more spg 6/7/2022 + availablekeys = len(self.oauths) + keytouse = randint(0,availablekeys-1) if platform == 'github': - self.headers = {'Authorization': 'token %s' % self.oauths[0]['access_token']} + self.headers = {'Authorization': 'token %s' % self.oauths[keytouse]['access_token']} elif platform == 'gitlab': - self.headers = {'Authorization': 'Bearer %s' % self.oauths[0]['access_token']} + self.headers = {'Authorization': 'Bearer %s' % self.oauths[keytouse]['access_token']} self.logger.info("OAuth initialized\n") From 96f5dbcda7621da864f176a2c68e58c25c9788ba Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 7 Jun 2022 13:27:59 -0500 Subject: [PATCH 3/8] added random library to setup.py --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index d70d3ef667..9edec98451 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ ], install_requires=[ "wheel", + "random", "coloredlogs==15.0", "Beaker==1.11.0", "SQLAlchemy==1.3.23", From 29f66fd227a9ef6399487e7d386550de57c26856 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Tue, 7 Jun 2022 13:56:50 -0500 Subject: [PATCH 4/8] playing with api key timeouts. --- workers/worker_git_integration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workers/worker_git_integration.py b/workers/worker_git_integration.py index 8399822bf5..18fc93fccf 100644 --- a/workers/worker_git_integration.py +++ b/workers/worker_git_integration.py @@ -219,7 +219,8 @@ def init_oauths(self, platform='github'): self.logger.debug('in github oauth block') elif platform == 'gitlab': self.headers = {'Authorization': 'Bearer %s' % oauth['access_token']} - response = requests.get(url=url, headers=self.headers, timeout=180) + ## Changed timeout from 180 to 12. Seems to be hanging in some workers. + response = requests.get(url=url, headers=self.headers, timeout=12) self.oauths.append({ 'oauth_id': oauth['oauth_id'], 'access_token': oauth['access_token'], From 98b9344302999076a3ddc117e70b0bca841e4fb0 Mon Sep 17 00:00:00 2001 From: Sean Goggins Date: Wed, 8 Jun 2022 17:41:45 -0500 Subject: [PATCH 5/8] update metadata and setup Signed-off-by: Sean Goggins --- metadata.py | 4 ++-- setup.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/metadata.py b/metadata.py index d529a239db..98a770e80a 100644 --- a/metadata.py +++ b/metadata.py @@ -5,8 +5,8 @@ __short_description__ = "Python 3 package for free/libre and open-source software community metrics, models & data collection" -__version__ = "0.26.5" -__release__ = "v0.26.5" +__version__ = "0.26.7" +__release__ = "v0.26.7" __license__ = "MIT" __copyright__ = "University of Missouri, University of Nebraska-Omaha, CHAOSS, & Augurlabs 2022" diff --git a/setup.py b/setup.py index 9edec98451..d70d3ef667 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,6 @@ ], install_requires=[ "wheel", - "random", "coloredlogs==15.0", "Beaker==1.11.0", "SQLAlchemy==1.3.23", From f8fed1ccd7e6b472453789516901159e7e108e04 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Wed, 8 Jun 2022 18:14:17 -0500 Subject: [PATCH 6/8] materialized view update --- schema/generate/100-schema_update_102.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schema/generate/100-schema_update_102.sql b/schema/generate/100-schema_update_102.sql index db316fd7db..d1a822bb19 100644 --- a/schema/generate/100-schema_update_102.sql +++ b/schema/generate/100-schema_update_102.sql @@ -218,8 +218,8 @@ CREATE MATERIALIZED VIEW "augur_data"."explorer_entry_list" AS SELECT DISTINCT r.repo_git, rg.rg_name - FROM (repo r - JOIN repo_groups rg ON ((rg.repo_group_id = r.repo_group_id))) + FROM (augur_data.repo r + JOIN augur_data.repo_groups rg ON ((rg.repo_group_id = r.repo_group_id))) ORDER BY rg.rg_name; ALTER MATERIALIZED VIEW "augur_data"."explorer_entry_list" OWNER TO "augur"; From ddc38a986183ad30bd96ae793b0ca58bf8614c14 Mon Sep 17 00:00:00 2001 From: "Sean P. Goggins" Date: Thu, 9 Jun 2022 14:42:37 -0500 Subject: [PATCH 7/8] removed timeout. --- workers/worker_git_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workers/worker_git_integration.py b/workers/worker_git_integration.py index 18fc93fccf..7b20c6c68b 100644 --- a/workers/worker_git_integration.py +++ b/workers/worker_git_integration.py @@ -220,7 +220,7 @@ def init_oauths(self, platform='github'): elif platform == 'gitlab': self.headers = {'Authorization': 'Bearer %s' % oauth['access_token']} ## Changed timeout from 180 to 12. Seems to be hanging in some workers. - response = requests.get(url=url, headers=self.headers, timeout=12) + response = requests.get(url=url, headers=self.headers) self.oauths.append({ 'oauth_id': oauth['oauth_id'], 'access_token': oauth['access_token'], From a67acebe782d844352b90df6c9dd4243f147604d Mon Sep 17 00:00:00 2001 From: Sean Goggins Date: Fri, 10 Jun 2022 16:05:10 -0500 Subject: [PATCH 8/8] contributor commit processing count checker by repo Signed-off-by: Sean Goggins --- user-resolution-rate-check.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 user-resolution-rate-check.sh diff --git a/user-resolution-rate-check.sh b/user-resolution-rate-check.sh new file mode 100755 index 0000000000..5e64e88a1c --- /dev/null +++ b/user-resolution-rate-check.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cat logs/workers/facade_worker/facade_worker_*_collection.log | grep "Processing repo contributors for repo:" | wc -l