Merge pull request #2872 from chaoss/release-fixes

Address release bugs

augur/tasks/github/messages.py

@@ -4,7 +4,7 @@
 from augur.tasks.init.celery_app import celery_app as celery
 from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask
 from augur.application.db.data_parse import *
-from augur.tasks.github.util.github_data_access import GithubDataAccess
+from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException
 from augur.tasks.github.util.github_task_session import GithubTaskManifest
 from augur.tasks.util.worker_util import remove_duplicate_dicts
 from augur.tasks.github.util.util import get_owner_repo
@@ -97,7 +97,11 @@ def process_large_issue_and_pr_message_collection(repo_id, repo_git: str, logger
     all_data = []
     for comment_url in comment_urls:
 
-        messages = list(github_data_access.paginate_resource(comment_url))
+        try:
+            messages = list(github_data_access.paginate_resource(comment_url))
+        except UrlNotFoundException as e:
+            logger.warning(e)
+            continue
 
         all_data += messages
 

augur/tasks/github/pull_requests/tasks.py

@@ -5,8 +5,7 @@
 from augur.tasks.init.celery_app import celery_app as celery
 from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask, AugurSecondaryRepoCollectionTask
 from augur.application.db.data_parse import *
-from augur.tasks.github.util.github_data_access import GithubDataAccess
-from augur.tasks.github.util.github_paginator import GithubPaginator
+from augur.tasks.github.util.github_data_access import GithubDataAccess, UrlNotFoundException
 from augur.tasks.util.worker_util import remove_duplicate_dicts
 from augur.tasks.github.util.util import add_key_value_pair_to_dicts, get_owner_repo
 from augur.application.db.models import PullRequest, Message, PullRequestReview, PullRequestLabel, PullRequestReviewer, PullRequestMeta, PullRequestAssignee, PullRequestReviewMessageRef, Contributor, Repo
@@ -359,8 +358,12 @@ def collect_pull_request_reviews(repo_git: str, full_collection: bool) -> None:
 
             pr_review_url = f"https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}/reviews"
 
-            pr_reviews = list(github_data_access.paginate_resource(pr_review_url))
-
+            try:
+                pr_reviews = list(github_data_access.paginate_resource(pr_review_url))
+            except UrlNotFoundException as e:
+                logger.warning(e)
+                continue
+
             if pr_reviews:
                 all_pr_reviews[pull_request_id] = pr_reviews
 

augur/tasks/github/util/github_graphql_data_access.py

@@ -145,6 +145,10 @@ def __extract_data_section(self, keys, json_response):
 
         # iterate deeper into the json_response object until we get to the desired data
         for value in keys:
+
+            if core is None:
+                raise Exception(f"Error: 'core' is None when trying to index by {value}. Response: {json_response}")
+
             core = core[value]
 
         return core