Skip to content

Commit

Permalink
Merge pull request #1559 from chaoss/andrew-print-traceback-method
Browse files Browse the repository at this point in the history 
Add print traceback method to pr worker
  • Loading branch information
@sgoggins
sgoggins authored Jan 13, 2022
2 parents 9a633e1 + bf59e48 commit 966698e
Showing 1 changed file with 38 additions and 76 deletions.
114 changes: 38 additions & 76 deletions workers/pull_request_worker/pull_request_worker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,9 +373,7 @@ def pull_request_commits_model(self, task_info, repo_id):
f"Inserted Pull Request Commit: {result.inserted_primary_key}\n"
)
except Exception as e:
self.logger.debug(f"pr_commit exception registered: {e}.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
self.print_traceback("pr commits model", e)
continue

self.register_task_completion(self.task_info, self.repo_id, 'pull_request_commits')
Expand Down Expand Up @@ -428,15 +426,10 @@ def pk_source_increment_insert(inc_source_prs, action_map):
}
}, prefix='user.'
)
except Exception as e:
self.logger.debug(f"Pull Requests model failed with {e}.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")

except Exception as e:
self.print_traceback("pull requests model", e)
else:
self.logger.info("Contributor enrichment is not needed, no inserts in action map.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")

prs_insert = []
try:
Expand Down Expand Up @@ -497,14 +490,8 @@ def pk_source_increment_insert(inc_source_prs, action_map):
'data_source': 'GitHub API'
} for pr in inc_source_prs['insert']
]
except Exception as e:
self.logger.debug(f"Pull Requests model failed with {e}.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")


# Removed due to error 11/18/2021 : and 'title' in pr['milestone']

except Exception as e:
self.print_traceback("Extracting data from source in pr model", e)
#The b_pr_src_id bug comes from here
'''
9/20/2021: Put the method definition for bulk insert here for reference. The method
Expand Down Expand Up @@ -590,6 +577,13 @@ def bulk_insert(

return pk_source_prs

def print_traceback(self, exception_message, exception, debug_log=True):

if debug_log:
self.logger.debug(f"{exception_message}. ERROR: {exception}", exc_info=sys.exc_info())
else:
self.logger.info(f"{exception_message}. ERROR: {exception}", exc_info=sys.exc_info())

def pull_requests_model(self, entry_info, repo_id):
"""Pull Request data collection function. Query GitHub API for PhubRs.
Expand All @@ -608,45 +602,34 @@ def pull_requests_model(self, entry_info, repo_id):

try:
pk_source_prs = self._get_pk_source_prs()
except Exception as e:
self.logger.debug(f"Pull Requests model failed with {e}.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")


except Exception as e:
self.print_traceback("Pull Requests model", e)
#self.write_debug_data(pk_source_prs, 'pk_source_prs')

if pk_source_prs:
try:
self.pull_request_comments_model(pk_source_prs)
self.logger.info(f"Pull request comments model.")
except Exception as e:
self.logger.debug(f"PR comments model failed on {e}. exception registered.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
except Exception as e:
self.print_traceback("PR comments model", e)
finally:
try:
self.pull_request_events_model(pk_source_prs)
self.logger.info(f"Pull request events model.")
except Exception as e:
self.logger.debug(f"PR events model failed on {e}. exception registered for pr_step.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
except Exception as e:
self.print_traceback("PR events model", e)

finally:
try:
self.logger.info(f"Pull request reviews model factored out for now due to speed.")
except Exception as e:
self.logger.debug(f"PR reviews model, which is factored out for now due to speed, somehow managed to fail on {e}. exception registered for pr_step.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
except Exception as e:
self.print_traceback("PR reviews model, which is factored out for now due to speed", e)
finally:
try:
self.pull_request_nested_data_model(pk_source_prs)
self.logger.info(f"Pull request nested data model.")
except Exception as e:
self.logger.debug(f"PR nested model failed on {e}. exception registered for pr_step.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
except Exception as e:
self.print_traceback("PR nested model", e)
finally:
self.logger.debug("finished running through four models.")

Expand Down Expand Up @@ -715,17 +698,14 @@ def pr_comments_insert(inc_pr_comments, comment_action_map, comment_ref_action_m
'platform_node_id': comment['node_id']
} for comment in inc_pr_comments['insert']
]
except Exception as e:
self.logger.debug(f"Comments model issue/error: {e}. exception registered.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
except Exception as e:
self.print_traceback("Creting list of pr comments", e)

try:
self.bulk_insert(self.message_table, insert=pr_comments_insert,
unique_columns=comment_action_map['insert']['augur'])
except Exception as e:
self.logger.debug(f"PR comments data model failed on {e}. exception registered.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
except Exception as e:
self.print_traceback("Bulk inserting pr comments", e)
finally:
try:
c_pk_source_comments = self.enrich_data_primary_keys(
Expand Down Expand Up @@ -773,17 +753,10 @@ def pr_comments_insert(inc_pr_comments, comment_action_map, comment_ref_action_m

self.bulk_insert(self.pull_request_message_ref_table, insert=pr_message_ref_insert,
unique_columns=comment_ref_action_map['insert']['augur'])

except Exception as e:

self.logger.info(f"message inserts failed with: {e}.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")

self.print_traceback("Gathering and inserting data into pr mesaage ref table", e)
finally:

self.logger.info("Finished message insert section.")

# TODO: add relational table so we can include a where_clause here
try:
pr_comments = self.paginate_endpoint(
Expand All @@ -804,14 +777,11 @@ def pr_comments_insert(inc_pr_comments, comment_action_map, comment_ref_action_m
stagger=True,
insertion_method=pr_comments_insert
)

pr_comments_insert(pr_comments,comment_action_map,comment_ref_action_map)
self.logger.info(f"comments inserted for repo_id: {self.repo_id}")
return
except Exception as e:
self.logger.info(f"exception registered in paginate endpoint for issue comments: {e}")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
self.print_traceback("Staggered paginate endpoint for pr comments", e)
finally:
self.logger.debug(f"Pull request messages and message refs worked without exception for {self.repo_id}")

Expand Down Expand Up @@ -882,10 +852,8 @@ def pull_request_events_model(self, pk_source_prs=[]):
]
try:
self.bulk_insert(self.pull_request_events_table, insert=pr_events_insert, unique_columns=event_action_map['insert']['augur'])
except Exception as e:
self.logger.debug(f"PR events data model failed on {e}. exception registered.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
except Exception as e:
self.print_traceback("Bulk insert pr events", e)

def pull_request_nested_data_model(self, pk_source_prs=[]):
try:
Expand All @@ -897,12 +865,8 @@ def pull_request_nested_data_model(self, pk_source_prs=[]):
else:
#prdata = json.loads(json.dumps(pk_source_prs))
self.logger.debug("nested model loaded.")

except Exception as e:

self.logger.debug(f'gettign source prs failed for nested model on {e}.')


except Exception as e:
self.print_traceback("Getting source prs in nested pr model", e)
labels_all = []
reviewers_all = []
assignees_all = []
Expand Down Expand Up @@ -1112,10 +1076,8 @@ def pull_request_nested_data_model(self, pk_source_prs=[]):
] # reverted above to see if it works with other fixes.
self.bulk_insert(self.pull_request_meta_table, insert=meta_insert)

except Exception as e:
self.logger.debug(f"Nested Model error at loop {pr_nested_loop} : {e}.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
except Exception as e:
self.print_traceback(f"Nested model error at loop {pr_nested_loop}", e)
continue

def query_pr_repo(self, pr_repo, pr_repo_type, pr_meta_id):
Expand Down Expand Up @@ -1167,8 +1129,8 @@ def query_pr_repo(self, pr_repo, pr_repo_type, pr_meta_id):
self.logger.info(
f"Finished adding PR {pr_repo_type} Repo data for PR with id {self.pr_id_inc}"
)
except Exception as e:
except Exception as e:
self.logger.debug(f"repo exception registerred for PRs: {e}")
self.logger.debug(f"Nested Model error at loop {pr_nested_loop} : {e}.")
stacker = traceback.format_exc()
self.logger.debug(f"{stacker}")
self.logger.debug(f"{stacker}")

0 comments on commit 966698e

Please sign in to comment.