diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000000..c23bfd7bb3 --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,31 @@ +name: "run-linting-checks" +on: + pull_request: + branches: [main, dev] + +jobs: + run-pylint: + name: runner / pylint + permissions: write-all + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: dciborow/action-pylint@0.1.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-review + level: warning + glob_pattern: "**/*.py" + filter_mode: "file" + + misspell: + name: runner / misspell + runs-on: ubuntu-latest + steps: + - name: Highlight any misspellings in changes. + uses: actions/checkout@v4 + - name: misspell + uses: reviewdog/action-misspell@v1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + locale: "US" \ No newline at end of file diff --git a/.pylintrc b/.pylintrc index 0b1b7d2049..0056af873b 100644 --- a/.pylintrc +++ b/.pylintrc @@ -12,7 +12,7 @@ #refactoring checker #enable=R -disable=E0611,E1101,W1203,R0801,W0614,W0611,C0411,C0103,C0301,C0303,C0304,C0305,W0311 +disable=E0611,E1101,W1203,R0801,W0614,W0611,C0411,C0103,C0301,C0303,C0304,C0305,W0311,E0401 # Analyse import fallback blocks. This can be used to support both Python 2 and diff --git a/augur/api/routes/util.py b/augur/api/routes/util.py index 1f95b9b7a2..71d3526b96 100644 --- a/augur/api/routes/util.py +++ b/augur/api/routes/util.py @@ -1,10 +1,11 @@ #SPDX-License-Identifier: MIT +from augur.api.routes import AUGUR_API_VERSION +from ..server import app, engine import base64 import sqlalchemy as s import pandas as pd import json from flask import Response -import logging from augur.application.db.session import DatabaseSession from augur.application.logs import AugurLogger @@ -12,10 +13,6 @@ logger = AugurLogger("augur").get_logger() -from augur.api.routes import AUGUR_API_VERSION -from ..server import app, engine - - @app.route('/{}/repo-groups'.format(AUGUR_API_VERSION)) def get_all_repo_groups(): #TODO: make this name automatic - wrapper? repoGroupsSQL = s.sql.text(""" @@ -54,9 +51,9 @@ def get_all_repos(): (select * from api_get_all_repos_issues) b on repo.repo_id = b.repo_id - left outer join - (select * from api_get_all_repo_prs) c - on repo.repo_id=c.repo_id + left outer join + (select * from api_get_all_repo_prs) c + on repo.repo_id=c.repo_id JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id order by repo_name """) @@ -95,9 +92,9 @@ def get_repos_in_repo_group(repo_group_id): (select * from api_get_all_repos_issues) b on repo.repo_id = b.repo_id - left outer join - (select * from api_get_all_repo_prs) c - on repo.repo_id=c.repo_id + left outer join + (select * from api_get_all_repo_prs) c + on repo.repo_id=c.repo_id JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id WHERE repo_groups.repo_group_id = :repo_group_id @@ -111,6 +108,49 @@ def get_repos_in_repo_group(repo_group_id): status=200, mimetype="application/json") +@app.route('/{}/repos/'.format(AUGUR_API_VERSION)) +def get_repo_by_id(repo_id: int) -> Response: + repo_by_id_SQL = s.sql.text(""" + SELECT + repo.repo_id, + repo.repo_name, + repo.description, + repo.repo_git AS url, + a.commits_all_time, + b.issues_all_time, + c.pull_requests_all_time, + rg_name, + repo.repo_group_id + FROM + repo + LEFT OUTER JOIN + (SELECT * FROM api_get_all_repos_commits) a + ON repo.repo_id = a.repo_id + LEFT OUTER JOIN + (SELECT * FROM api_get_all_repos_issues) b + ON repo.repo_id = b.repo_id + LEFT OUTER JOIN + (SELECT * FROM api_get_all_repo_prs) c + ON repo.repo_id = c.repo_id + JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id + WHERE + repo.repo_id = :id + """) + + results = pd.read_sql(repo_by_id_SQL, engine, params={"id": repo_id}) + results["url"] = results["url"].apply(lambda datum: datum.split("//")[1]) # cut "https://" off the URL + results["base64_url"] = [base64.b64encode(results.at[i, "url"].encode()) for i in results.index] + data = results.to_json(orient="records", date_format="iso", date_unit="ms") + + if not data or data == "[]": + return Response(response='{"status": "Repository ' + str(repo_id) + ' does not exist"}', + status=400, + mimetype="application/json") + + return Response(response=data[1:-1], # cut off brackets at each end, turns list of length 1 into single value + status=200, + mimetype="application/json") + @app.route('/{}/owner//repo/'.format(AUGUR_API_VERSION)) def get_repo_by_git_name(owner, repo): diff --git a/augur/api/view/routes.py b/augur/api/view/routes.py index bf6e8fc056..72164a9291 100644 --- a/augur/api/view/routes.py +++ b/augur/api/view/routes.py @@ -1,4 +1,8 @@ +""" +Defines the api routes for the augur views +""" import logging +import math from flask import Flask, render_template, render_template_string, request, abort, jsonify, redirect, url_for, session, flash from sqlalchemy.orm.exc import NoResultFound from .utils import * @@ -37,9 +41,9 @@ def root(path=""): def logo(brand=None): if brand is None: return redirect(url_for('static', filename='img/augur_logo.png')) - elif "augur" in brand: + if "augur" in brand: return logo(None) - elif "chaoss" in brand: + if "chaoss" in brand: return redirect(url_for('static', filename='img/Chaoss_Logo_white.png')) return "" diff --git a/augur/api/view/utils.py b/augur/api/view/utils.py index 7712873b55..298e9950ae 100644 --- a/augur/api/view/utils.py +++ b/augur/api/view/utils.py @@ -1,6 +1,10 @@ +""" +Defines utility functions used by the augur api views +""" from pathlib import Path from concurrent.futures import ThreadPoolExecutor from flask import render_template, flash, url_for, Flask +from .init import init_logging from .init import * from ..server import app, db_session from augur.application.config import AugurConfig diff --git a/augur/application/db/data_parse.py b/augur/application/db/data_parse.py index 2d5c51a899..7562181398 100644 --- a/augur/application/db/data_parse.py +++ b/augur/application/db/data_parse.py @@ -39,6 +39,20 @@ def extract_needed_pr_label_data(labels: List[dict], repo_id: int, tool_source: def extract_needed_mr_label_data(labels: List[dict], repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for mr label data from the api response + + Arguments: + labels: List of dictionaries of label data + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + List of parsed label dicts + """ if len(labels) == 0: return [] @@ -65,8 +79,21 @@ def extract_needed_mr_label_data(labels: List[dict], repo_id: int, tool_source: return label_dicts -# retrieve only the needed data for pr assignees from the api response def extract_needed_pr_assignee_data(assignees: List[dict], repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for pr assignees from the api response + + Arguments: + assignees: List of dictionaries of asignee data + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + List of parsed asignee dicts + """ if len(assignees) == 0: return [] @@ -89,6 +116,20 @@ def extract_needed_pr_assignee_data(assignees: List[dict], repo_id: int, tool_so return assignee_dicts def extract_needed_merge_request_assignee_data(assignees: List[dict], repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for merge request assignees from the api response + + Arguments: + assignees: List of dictionaries of asignee data + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + List of parsed asignee dicts + """ if len(assignees) == 0: return [] @@ -112,8 +153,21 @@ def extract_needed_merge_request_assignee_data(assignees: List[dict], repo_id: i -# retrieve only the needed data for pr reviewers from the api response def extract_needed_pr_reviewer_data(reviewers: List[dict], repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for pr reviewers from the api response + + Arguments: + reviewers: List of dictionaries of reviewer data + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + List of parsed reviewer dicts + """ if len(reviewers) == 0: return [] @@ -299,6 +353,20 @@ def extract_needed_issue_assignee_data(assignees: List[dict], repo_id: int, tool return assignee_dicts def extract_needed_gitlab_issue_assignee_data(assignees: List[dict], repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for gitlab issue assignees from the api response + + Arguments: + assignees: List of dictionaries of gitlab assignee data + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + List of parsed assignee dicts + """ if len(assignees) == 0: return [] @@ -351,6 +419,20 @@ def extract_needed_issue_label_data(labels: List[dict], repo_id: int, tool_sourc def extract_needed_gitlab_issue_label_data(labels: List[dict], repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for gitlab issue labels from the api response + + Arguments: + labels: List of dictionaries of gitlab issue label data + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + List of parsed label dicts + """ if len(labels) == 0: return [] @@ -376,8 +458,22 @@ def extract_needed_gitlab_issue_label_data(labels: List[dict], repo_id: int, too -# retrieve only the needed data for pr labels from the api response def extract_needed_issue_message_ref_data(message: dict, issue_id: int, repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for pr labels from the api response + + Arguments: + message: Message data dict + issue_id: id of the issue + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + Dict of message ref data. + """ message_ref_dict = { 'issue_id': issue_id, @@ -409,7 +505,19 @@ def extract_needed_pr_message_ref_data(comment: dict, pull_request_id: int, repo def extract_needed_pr_data(pr, repo_id, tool_source, tool_version): + """ + Retrieve only the needed data for the pr api response + + Arguments: + pr: PR data dict + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + + Returns: + Parsed pr dict + """ pr = { 'repo_id': repo_id, @@ -468,6 +576,20 @@ def extract_needed_pr_data(pr, repo_id, tool_source, tool_version): return pr def extract_needed_issue_data(issue: dict, repo_id: int, tool_source: str, tool_version: str, data_source: str): + """ + Retrieve only the needed data for the issue api response + + Arguments: + issue: Issue data dict + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: platform source + + + Returns: + Parsed issue dict + """ dict_data = { 'cntrb_id': None, # this the contributor who closed the issue @@ -612,6 +734,19 @@ def extract_needed_pr_review_data(review, pull_request_id, repo_id, platform_id, return review_row def extract_needed_pr_data_from_gitlab_merge_request(pr, repo_id, tool_source, tool_version): + """ + Retrieve only the needed data for the pr gitlab api response + + Arguments: + pr: PR data dict + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + + + Returns: + Parsed pr dict + """ pr_dict = { 'repo_id': repo_id, @@ -659,6 +794,20 @@ def extract_needed_pr_data_from_gitlab_merge_request(pr, repo_id, tool_source, t def extract_needed_issue_data_from_gitlab_issue(issue: dict, repo_id: int, tool_source: str, tool_version: str, data_source: str): + """ + Retrieve only the needed data for the issue gitlab api response + + Arguments: + issue: Issue data dict + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: platform source + + + Returns: + Parsed issue dict + """ issue_dict = { "repo_id": repo_id, @@ -692,6 +841,22 @@ def extract_needed_issue_data_from_gitlab_issue(issue: dict, repo_id: int, tool_ def extract_gitlab_mr_event_data(event: dict, pr_id: int, platform_id: int, repo_id: int, tool_source: str, tool_version: str, data_source: str) -> dict: + """ + Retrieve only the needed data for the mr event gitlab api response + + Arguments: + event: Event data dict + pr_id: id of the pr + platform_id: id of the platform + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: platform source + + + Returns: + Parsed event dict + """ mr_event = { 'pull_request_id': pr_id, @@ -712,6 +877,22 @@ def extract_gitlab_mr_event_data(event: dict, pr_id: int, platform_id: int, repo return mr_event def extract_gitlab_issue_event_data(event: dict, issue_id: int, platform_id: int, repo_id: int, tool_source: str, tool_version: str, data_source: str) -> dict: + """ + Retrieve only the needed data for the issue event gitlab api response + + Arguments: + event: Event data dict + issue_id: id of the issue + platform_id: id of the platform + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: platform source + + + Returns: + Parsed event dict + """ issue_event = { "issue_event_src_id": event['target_id'], @@ -732,8 +913,21 @@ def extract_gitlab_issue_event_data(event: dict, issue_id: int, platform_id: int return issue_event -# retrieve only the needed data for pr reviewers from the api response -def extract_needed_mr_reviewer_data(data: List[dict], pull_request_id, repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: +def extract_needed_mr_reviewer_data(data: List[dict], pull_request_id, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for pr reviewers from the api response + + Arguments: + data: List of dictionaries that contain mr reviewer data to parse + pull_request_id: id of the PR + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + List of extracted relevant data from needed mr reviwer data + """ if len(data) == 0: return [] @@ -741,7 +935,7 @@ def extract_needed_mr_reviewer_data(data: List[dict], pull_request_id, repo_id: reviewer_dicts = [] for x in data: - for reviewer in x["suggested_approvers"]: + for _ in x["suggested_approvers"]: reviewer_dict = { 'pull_request_id': pull_request_id, @@ -757,6 +951,21 @@ def extract_needed_mr_reviewer_data(data: List[dict], pull_request_id, repo_id: def extract_needed_mr_commit_data(commit, repo_id, pull_request_id, tool_source, tool_version, data_source): + """ + Retrieve only the needed data for mr commit data from the api response + + Arguments: + commit: commit data dictionary + repo_id: augur id of the repository + pull_request_id: id of the PR + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + Dictionary of the extracted commit data + """ commit = { 'pull_request_id': pull_request_id, @@ -773,7 +982,21 @@ def extract_needed_mr_commit_data(commit, repo_id, pull_request_id, tool_source, def extract_needed_mr_file_data(gitlab_file_data, repo_id, pull_request_id, tool_source, tool_version, data_source): + """ + Retrieve only the needed data for mr file data from the api response + + Arguments: + gitlab_file_data: file data dictionary + repo_id: augur id of the repository + pull_request_id: id of the PR + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + Returns: + List of dicts of parsed gitlab file changes + """ files = [] changes = gitlab_file_data["changes"] @@ -781,7 +1004,7 @@ def extract_needed_mr_file_data(gitlab_file_data, repo_id, pull_request_id, tool try: deletes = int(file_changes['diff'].split('@@')[1].strip().split(' ')[0].split(',')[1]) adds = int(file_changes['diff'].split('@@')[1].strip().split(' ')[1].split(',')[1]) - except: + except Exception: deletes = 0 adds = 0 @@ -802,7 +1025,21 @@ def extract_needed_mr_file_data(gitlab_file_data, repo_id, pull_request_id, tool def extract_needed_mr_metadata(mr_dict, repo_id, pull_request_id, tool_source, tool_version, data_source): + """ + Retrieve only the needed data for mr metadata from the api response + + Arguments: + mr_dict: mr data dictionary + repo_id: augur id of the repository + pull_request_id: id of the PR + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + Returns: + List of dicts of parsed mr metadata + """ head = {'sha': mr_dict['diff_refs']['head_sha'], 'ref': mr_dict['target_branch'], 'label': str(mr_dict['target_project_id']) + ':' + mr_dict['target_branch'], @@ -841,6 +1078,22 @@ def extract_needed_mr_metadata(mr_dict, repo_id, pull_request_id, tool_source, t def extract_needed_gitlab_issue_message_ref_data(message: dict, issue_id: int, repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Extract the message id for a given message on an issue from an api response + and connect it to the relevant repo id. + + Arguments: + message: message data dict + issue_id: id of the issue + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + Dict containing the message ref id as well as the repo id. + """ message_ref_dict = { 'issue_id': issue_id, @@ -855,7 +1108,22 @@ def extract_needed_gitlab_issue_message_ref_data(message: dict, issue_id: int, r return message_ref_dict -def extract_needed_gitlab_message_data(comment: dict, platform_id: int, repo_id: int, tool_source: str, tool_version: str, data_source: str): +def extract_needed_gitlab_message_data(comment: dict, platform_id: int, tool_source: str, tool_version: str, data_source: str): + """ + Extract specific metadata for a comment from an api response + and connect it to the relevant platform id. + + Arguments: + comment: comment data dict + platform_id: augur id of the relevant platform + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + Dict containing parsed comment text and metadata + """ comment_dict = { "pltfrm_id": platform_id, @@ -870,8 +1138,23 @@ def extract_needed_gitlab_message_data(comment: dict, platform_id: int, repo_id: return comment_dict -# retrieve only the needed data for pr labels from the api response def extract_needed_gitlab_mr_message_ref_data(comment: dict, pull_request_id: int, repo_id: int, tool_source: str, tool_version: str, data_source: str) -> List[dict]: + """ + Retrieve only the needed data for pr labels from the api response + + Arguments: + comment: comment data dict + pull_request_id: id of the PR + repo_id: augur id of the repository + platform_id: augur id of the relevant platform + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + + Returns: + Dict containing the comment, pr and repo id of the parsed comment data. + """ pr_msg_ref = { 'pull_request_id': pull_request_id, diff --git a/augur/application/db/models/augur_data.py b/augur/application/db/models/augur_data.py index cfbcdca1d7..7f97e4bbdc 100644 --- a/augur/application/db/models/augur_data.py +++ b/augur/application/db/models/augur_data.py @@ -1088,6 +1088,7 @@ def insert_github_repo(session, url: str, repo_group_id: int, tool_source, repo_ Args: url: repo url repo_group_id: group to assign repo to + repo_type: github or gitlab Note: If repo row exists then it will update the repo_group_id if param repo_group_id is not a default. If it does not exist is will simply insert the repo. diff --git a/augur/application/db/session.py b/augur/application/db/session.py index f1d1e64dd0..22379ad050 100644 --- a/augur/application/db/session.py +++ b/augur/application/db/session.py @@ -194,14 +194,15 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s except Exception as e: #self.logger.info(e) - if(len(data) == 1): + if len(data) == 1: raise e - else: - first_half = data[:len(data)//2] - second_half = data[len(data)//2:] + + time.sleep(3) + first_half = data[:len(data)//2] + second_half = data[len(data)//2:] - self.insert_data(first_half, table,natural_keys, return_columns, string_fields, on_conflict_update) - self.insert_data(second_half,table, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(first_half, table, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(second_half,table, natural_keys, return_columns, string_fields, on_conflict_update) else: self.logger.error("Unable to insert data in 10 attempts") @@ -231,14 +232,15 @@ def insert_data(self, data: Union[List[dict], dict], table, natural_keys: List[s raise e except Exception as e: - if(len(data) == 1): + if len(data) == 1: raise e - else: - first_half = data[:len(data)//2] - second_half = data[len(data)//2:] + + time.sleep(3) + first_half = data[:len(data)//2] + second_half = data[len(data)//2:] - self.insert_data(first_half, natural_keys, return_columns, string_fields, on_conflict_update) - self.insert_data(second_half, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(first_half, table, natural_keys, return_columns, string_fields, on_conflict_update) + self.insert_data(second_half, table, natural_keys, return_columns, string_fields, on_conflict_update) else: self.logger.error("Unable to insert and return data in 10 attempts") diff --git a/augur/tasks/github/messages/tasks.py b/augur/tasks/github/messages/tasks.py index 6e23434bae..4dfd3a634b 100644 --- a/augur/tasks/github/messages/tasks.py +++ b/augur/tasks/github/messages/tasks.py @@ -187,7 +187,8 @@ def process_messages(messages, task_name, repo_id, logger, augur_db): message_string_fields = ["msg_text"] message_return_data = augur_db.insert_data(message_dicts, Message, message_natural_keys, return_columns=message_return_columns, string_fields=message_string_fields) - + if message_return_data is None: + return pr_message_ref_dicts = [] issue_message_ref_dicts = [] diff --git a/augur/tasks/github/pull_requests/tasks.py b/augur/tasks/github/pull_requests/tasks.py index 478260bcd7..8db394754c 100644 --- a/augur/tasks/github/pull_requests/tasks.py +++ b/augur/tasks/github/pull_requests/tasks.py @@ -76,7 +76,16 @@ def retrieve_all_pr_data(repo_git: str, logger, key_auth) -> None: def process_pull_requests(pull_requests, task_name, repo_id, logger, augur_db): - + """ + Parse and insert all retrieved PR data. + + Arguments: + pull_requests: List of paginated pr endpoint data + task_name: Name of the calling task and the repo + repo_id: augur id of the repository + logger: logging object + augur_db: sqlalchemy db object + """ tool_source = "Pr Task" tool_version = "2.0" data_source = "Github API" diff --git a/augur/tasks/github/releases/core.py b/augur/tasks/github/releases/core.py index f3050fc1b3..5957d4cb57 100644 --- a/augur/tasks/github/releases/core.py +++ b/augur/tasks/github/releases/core.py @@ -84,7 +84,8 @@ def insert_release(augur_db, logger, repo_id, owner, release, tag_only = False): release_inf = get_release_inf(repo_id, release, tag_only) #Do an upsert - augur_db.insert_data(release_inf,Release,['release_id']) + string_fields = ["release_name", "release_description", "release_author", "release_tag_name"] + augur_db.insert_data(release_inf,Release,['release_id'], string_fields=string_fields) logger.info(f"Inserted info for {owner}/{repo_id}/{release['name']}\n") diff --git a/augur/tasks/github/util/util.py b/augur/tasks/github/util/util.py index 0400b82e1a..42989dcca3 100644 --- a/augur/tasks/github/util/util.py +++ b/augur/tasks/github/util/util.py @@ -58,6 +58,17 @@ def parse_json_response(logger: logging.Logger, response: httpx.Response) -> dic return json.loads(json.dumps(response.text)) def get_repo_weight_by_issue(logger,repo_git): + """ + Retrieve the sum of the number of issues and prs in a repository from a graphql query. + + Arguments: + logger: logger object + repo_git: repository url + + Returns: + Sum of issues and prs for that repo + """ + from augur.tasks.github.util.gh_graphql_entities import GitHubRepo as GitHubRepoGraphql owner,name = get_owner_repo(repo_git) diff --git a/augur/tasks/gitlab/events_task.py b/augur/tasks/gitlab/events_task.py index 4224988b9f..8058831ba3 100644 --- a/augur/tasks/gitlab/events_task.py +++ b/augur/tasks/gitlab/events_task.py @@ -1,3 +1,6 @@ +""" +Module to define the task methods to collect gitlab event data for augur +""" import logging from augur.tasks.init.celery_app import celery_app as celery @@ -13,6 +16,12 @@ @celery.task(base=AugurCoreRepoCollectionTask) def collect_gitlab_issue_events(repo_git) -> int: + """ + Retrieve and parse gitlab events for the desired repo + + Arguments: + repo_git: the repo url string + """ owner, repo = get_owner_repo(repo_git) @@ -36,6 +45,13 @@ def collect_gitlab_issue_events(repo_git) -> int: @celery.task(base=AugurCoreRepoCollectionTask) def collect_gitlab_merge_request_events(repo_git) -> int: + """ + Retrieve and parse gitlab mrs for the desired repo + + Arguments: + repo_git: the repo url string + """ + owner, repo = get_owner_repo(repo_git) @@ -57,13 +73,22 @@ def collect_gitlab_merge_request_events(repo_git) -> int: logger.info(f"{owner}/{repo} has no gitlab merge request events") -def retrieve_all_gitlab_event_data(type, repo_git, logger, key_auth) -> None: +def retrieve_all_gitlab_event_data(gtype, repo_git, logger, key_auth) -> None: + """ + Retrieve only the needed data for mr label data from the api response + + Arguments: + gtype: type of event data + repo_git: url of the relevant repo + logger: loggin object + key_auth: key auth cache and rotator object + """ owner, repo = get_owner_repo(repo_git) logger.info(f"Collecting gitlab issue events for {owner}/{repo}") - url = f"https://gitlab.com/api/v4/projects/{owner}%2f{repo}/events?target_type={type}" + url = f"https://gitlab.com/api/v4/projects/{owner}%2f{repo}/events?target_type={gtype}" events = GitlabApiHandler(key_auth, logger) all_data = [] @@ -75,18 +100,28 @@ def retrieve_all_gitlab_event_data(type, repo_git, logger, key_auth) -> None: if len(page_data) == 0: logger.debug( - f"{owner}/{repo}: Gitlab {type} Events Page {page} contains no data...returning") - logger.info(f"{owner}/{repo}: {type} Events Page {page} of {num_pages}") + f"{owner}/{repo}: Gitlab {gtype} Events Page {page} contains no data...returning") + logger.info(f"{owner}/{repo}: {gtype} Events Page {page} of {num_pages}") return all_data - logger.info(f"{owner}/{repo}: Gitlab {type} Events Page {page} of {num_pages}") + logger.info(f"{owner}/{repo}: Gitlab {gtype} Events Page {page} of {num_pages}") all_data += page_data return all_data def process_issue_events(events, task_name, repo_id, logger, augur_db): - + """ + Retrieve only the needed data for mr label data from the api response + + Arguments: + events: List of dictionaries of issue event data + task_name: name of the task as well as the repo being processed + repo_id: augur id of the repo + logger: logging object + augur_db: sqlalchemy db object + """ + tool_source = "Gitlab issue events task" tool_version = "2.0" data_source = "Gitlab API" @@ -122,7 +157,21 @@ def process_issue_events(events, task_name, repo_id, logger, augur_db): def process_mr_events(events, task_name, repo_id, logger, augur_db): + """ + Retrieve only the needed data for mr events from the api response + + Arguments: + labels: List of dictionaries of label data + repo_id: augur id of the repository + tool_source: The part of augur that processed the data + tool_version: The version of the augur task that processed the data + data_source: The source of the data + + Returns: + List of parsed label dicts + """ + tool_source = "Gitlab mr events task" tool_version = "2.0" data_source = "Gitlab API" diff --git a/augur/tasks/gitlab/gitlab_api_handler.py b/augur/tasks/gitlab/gitlab_api_handler.py index 8f111d3c48..5303d606e9 100644 --- a/augur/tasks/gitlab/gitlab_api_handler.py +++ b/augur/tasks/gitlab/gitlab_api_handler.py @@ -1,4 +1,7 @@ - +""" +Defines a GitlabApiHandler class to paginate and handle interaction with GitLab's +api through automatic use of relevant key auth and pagination tools. +""" import httpx import time import logging @@ -61,7 +64,7 @@ def get_length(self, url): issue_len = len(issues) """ - num_pages = self.get_num_pages() + num_pages = self.get_num_pages(url) self.logger.info(f"Num pages: {num_pages}") @@ -255,6 +258,16 @@ def get_num_pages(self, url) -> Optional[int]: return num_pages def hit_api(self, url, timeout, method): + """Attempt to retrieve data at given url. + + Args: + url: The url to retrieve the data from + timeout: time to wait until timeout + method: GET, POST, etc. + + Returns + The response object from hitting the url and the data on the page + """ return hit_api(self.key_manager, url, self.logger, timeout, method=method) @@ -273,7 +286,7 @@ def _set_paginaton_query_params(self, url): ################################################################################ -# Url Helper Method to remove query paramaters from the url +# Url Helper Method to remove query parameters from the url def clean_url(url: str, keys: List[str]) -> str: """Remove query params from url. @@ -300,7 +313,7 @@ def add_query_params(url: str, additional_params: dict) -> str: Args: url: the url that is being modified - additional_params: key value pairs specififying the paramaters to be added + additional_params: key value pairs specifying the parameters to be added Returns: The url with the key value pairs in additional_params added as query params @@ -370,4 +383,4 @@ def hit_api(key_manager, url: str, logger: logging.Logger, timeout: float = 10, time.sleep(round(timeout*1.5)) return None - return response \ No newline at end of file + return response diff --git a/augur/tasks/gitlab/gitlab_api_key_handler.py b/augur/tasks/gitlab/gitlab_api_key_handler.py index 50efa446f8..20bc1219ca 100644 --- a/augur/tasks/gitlab/gitlab_api_key_handler.py +++ b/augur/tasks/gitlab/gitlab_api_key_handler.py @@ -1,3 +1,8 @@ +""" +Defines the handler logic needed to effectively fetch GitLab auth keys +from either the redis cache or the database. Follows the same patterns as +the github api key handler. +""" import httpx import time import random @@ -11,7 +16,7 @@ class NoValidKeysError(Exception): - pass + """Defines an exception that is thrown when no gitlab keys are valid""" class GitlabApiKeyHandler(): @@ -102,7 +107,9 @@ def get_api_keys(self) -> List[str]: try: keys = self.get_api_keys_from_database() break - except: + except Exception as e: + self.logger.error(f"Ran into issue when fetching key from database:\n {e}\n") + self.logger.error("Sleeping for 5 seconds...") time.sleep(5) attempts += 1 @@ -135,7 +142,7 @@ def get_api_keys(self) -> List[str]: # shuffling the keys so not all processes get the same keys in the same order - valid_now = valid_keys + #valid_now = valid_keys #try: #self.logger.info(f'valid keys before shuffle: {valid_keys}') #valid_keys = random.sample(valid_keys, len(valid_keys)) diff --git a/augur/tasks/gitlab/gitlab_random_key_auth.py b/augur/tasks/gitlab/gitlab_random_key_auth.py index 86ad64b056..64ba31dd19 100644 --- a/augur/tasks/gitlab/gitlab_random_key_auth.py +++ b/augur/tasks/gitlab/gitlab_random_key_auth.py @@ -6,8 +6,8 @@ class GitlabRandomKeyAuth(RandomKeyAuth): - """Defines a github specific RandomKeyAuth class so - github collections can have a class randomly selects an api key for each request + """Defines a gitlab specific RandomKeyAuth class so + gitlab collections can have a class randomly selects an api key for each request """ def __init__(self, session: DatabaseSession, logger): diff --git a/augur/tasks/gitlab/gitlab_task_session.py b/augur/tasks/gitlab/gitlab_task_session.py index 1871e46c50..58a6e64373 100644 --- a/augur/tasks/gitlab/gitlab_task_session.py +++ b/augur/tasks/gitlab/gitlab_task_session.py @@ -1,14 +1,26 @@ +""" +Defines a GitLab-specific session and manifest object for use in GitLab tasks +""" from logging import Logger from augur.tasks.gitlab.gitlab_random_key_auth import GitlabRandomKeyAuth from augur.application.db.session import DatabaseSession class GitlabTaskManifest: + """ + Manifest object that represents the state and common elements of + the specified task. GitLab version for the GitLab tasks. + + Attributes: + augur_db: sqlalchemy db object + key_auth: GitLab specific key auth retrieval collection + logger: logging object + platform_id: GitLab specific platform id (github is 1) + """ def __init__(self, logger): from augur.tasks.init.celery_app import engine - from augur.application.db.session import DatabaseSession self.augur_db = DatabaseSession(logger, engine) self.key_auth = GitlabRandomKeyAuth(self.augur_db.session, logger) diff --git a/augur/tasks/gitlab/issues_task.py b/augur/tasks/gitlab/issues_task.py index 7f0c7787ee..cf6e5e5dab 100644 --- a/augur/tasks/gitlab/issues_task.py +++ b/augur/tasks/gitlab/issues_task.py @@ -1,3 +1,6 @@ +""" +Defines the set of tasks used to retrieve GitLab issue data. +""" import logging import traceback @@ -14,7 +17,12 @@ @celery.task(base=AugurCoreRepoCollectionTask) def collect_gitlab_issues(repo_git : str) -> int: + """ + Retrieve and parse gitlab issues for the desired repo + Arguments: + repo_git: the repo url string + """ logger = logging.getLogger(collect_gitlab_issues.__name__) with GitlabTaskManifest(logger) as manifest: @@ -45,6 +53,14 @@ def collect_gitlab_issues(repo_git : str) -> int: def retrieve_all_gitlab_issue_data(repo_git, logger, key_auth) -> None: + """ + Retrieve only the needed data for issues from the api response + + Arguments: + repo_git: url of the relevant repo + logger: loggin object + key_auth: key auth cache and rotator object + """ owner, repo = get_owner_repo(repo_git) @@ -73,7 +89,17 @@ def retrieve_all_gitlab_issue_data(repo_git, logger, key_auth) -> None: return all_data def process_issues(issues, task_name, repo_id, logger, augur_db) -> None: - + """ + Retrieve only the needed data for issues from the api response + + Arguments: + issues: List of dictionaries of issue data + task_name: name of the task as well as the repo being processed + repo_id: augur id of the repo + logger: logging object + augur_db: sqlalchemy db object + """ + # get repo_id or have it passed tool_source = "Gitlab Issue Task" tool_version = "2.0" @@ -153,6 +179,13 @@ def process_issues(issues, task_name, repo_id, logger, augur_db) -> None: @celery.task(base=AugurCoreRepoCollectionTask) def collect_gitlab_issue_comments(issue_ids, repo_git) -> int: + """ + Retrieve and parse gitlab events for the desired repo + + Arguments: + issue_ids: Set of issue ids to collect coments for + repo_git: repo url + """ owner, repo = get_owner_repo(repo_git) @@ -175,6 +208,15 @@ def collect_gitlab_issue_comments(issue_ids, repo_git) -> int: def retrieve_all_gitlab_issue_comments(key_auth, logger, issue_ids, repo_git): + """ + Retrieve only the needed data for issue comments + + Arguments: + key_auth: key auth cache and rotator object + logger: loggin object + issue_ids: ids of issues to find comements for + repo_git: repo url + """ owner, repo = get_owner_repo(repo_git) @@ -186,7 +228,7 @@ def retrieve_all_gitlab_issue_comments(key_auth, logger, issue_ids, repo_git): for id in issue_ids: - print(f"Collecting {owner}/{repo} gitlab issue comments for issue {index} of {issue_count}") + logger.info(f"Collecting {owner}/{repo} gitlab issue comments for issue {index} of {issue_count}") url = f"https://gitlab.com/api/v4/projects/{owner}%2f{repo}/issues/{id}/notes" @@ -206,6 +248,16 @@ def retrieve_all_gitlab_issue_comments(key_auth, logger, issue_ids, repo_git): def process_gitlab_issue_messages(data, task_name, repo_id, logger, augur_db): + """ + Retrieve only the needed data for issue messages from the api response + + Arguments: + data: List of dictionaries of issue event data + task_name: name of the task as well as the repo being processed + repo_id: augur id of the repo + logger: logging object + augur_db: sqlalchemy db object + """ tool_source = "Gitlab issue comments" tool_version = "2.0" @@ -238,7 +290,7 @@ def process_gitlab_issue_messages(data, task_name, repo_id, logger, augur_db): } message_dicts.append( - extract_needed_gitlab_message_data(message, platform_id, repo_id, tool_source, tool_version, data_source) + extract_needed_gitlab_message_data(message, platform_id, tool_source, tool_version, data_source) ) diff --git a/augur/tasks/gitlab/merge_request_task.py b/augur/tasks/gitlab/merge_request_task.py index 5672a79895..ccf3c7e012 100644 --- a/augur/tasks/gitlab/merge_request_task.py +++ b/augur/tasks/gitlab/merge_request_task.py @@ -13,6 +13,12 @@ @celery.task(base=AugurCoreRepoCollectionTask) def collect_gitlab_merge_requests(repo_git: str) -> int: + """ + Retrieve and parse gitlab MRs for the desired repo + + Arguments: + repo_git: the repo url string + """ logger = logging.getLogger(collect_gitlab_merge_requests.__name__) @@ -37,6 +43,14 @@ def collect_gitlab_merge_requests(repo_git: str) -> int: def retrieve_all_mr_data(repo_git: str, logger, key_auth) -> None: + """ + Retrieve only the needed data for MRs from the api response + + Arguments: + repo_git: url of the relevant repo + logger: loggin object + key_auth: key auth cache and rotator object + """ owner, repo = get_owner_repo(repo_git) @@ -66,6 +80,19 @@ def retrieve_all_mr_data(repo_git: str, logger, key_auth) -> None: def process_merge_requests(data, task_name, repo_id, logger, augur_db): + """ + Retrieve only the needed data for mr label data from the api response + + Arguments: + data: collection of mr data + task_name: name of the task as well as the repo being processed + repo_id: augur id of the repo + logger: logging object + augur_db: sqlalchemy db object + + Returns: + List of parsed MR ids. + """ tool_source = "Mr Task" tool_version = "2.0" @@ -129,6 +156,13 @@ def process_merge_requests(data, task_name, repo_id, logger, augur_db): @celery.task(base=AugurCoreRepoCollectionTask) def collect_merge_request_comments(mr_ids, repo_git) -> int: + """ + Retrieve and parse gitlab events for the desired repo + + Arguments: + mr_ids: ids of MRs to paginate comments for + repo_git: the repo url string + """ owner, repo = get_owner_repo(repo_git) @@ -152,6 +186,16 @@ def collect_merge_request_comments(mr_ids, repo_git) -> int: def process_gitlab_mr_messages(data, task_name, repo_id, logger, augur_db): + """ + Retrieve only the needed data for mr label data from the api response + + Arguments: + data: List of dictionaries of mr message data + task_name: name of the task as well as the repo being processed + repo_id: augur id of the repo + logger: logging object + augur_db: sqlalchemy db object + """ tool_source = "Gitlab mr comments" tool_version = "2.0" @@ -184,7 +228,7 @@ def process_gitlab_mr_messages(data, task_name, repo_id, logger, augur_db): } message_dicts.append( - extract_needed_gitlab_message_data(message, platform_id, repo_id, tool_source, tool_version, data_source) + extract_needed_gitlab_message_data(message, platform_id, tool_source, tool_version, data_source) ) @@ -214,6 +258,13 @@ def process_gitlab_mr_messages(data, task_name, repo_id, logger, augur_db): @celery.task(base=AugurCoreRepoCollectionTask) def collect_merge_request_metadata(mr_ids, repo_git) -> int: + """ + Retrieve and parse gitlab events for the desired repo + + Arguments: + mr_ids: list of mr ids to find metadata for + repo_git: the repo url string + """ owner, repo = get_owner_repo(repo_git) @@ -236,6 +287,16 @@ def collect_merge_request_metadata(mr_ids, repo_git) -> int: logger.info(f"{owner}/{repo} has no gitlab merge request metadata") def process_mr_metadata(data, task_name, repo_id, logger, augur_db): + """ + Retrieve only the needed data for mr label data from the api response + + Arguments: + data: List of dictionaries of mr metadata + task_name: name of the task as well as the repo being processed + repo_id: augur id of the repo + logger: logging object + augur_db: sqlalchemy db object + """ tool_source = "Mr Metadata Task" tool_version = "2.0" @@ -261,6 +322,13 @@ def process_mr_metadata(data, task_name, repo_id, logger, augur_db): @celery.task(base=AugurCoreRepoCollectionTask) def collect_merge_request_reviewers(mr_ids, repo_git) -> int: + """ + Retrieve and parse mr reviewers for the desired repo + + Arguments: + mr_ids: mrs to search for reviewers for + repo_git: the repo url string + """ owner, repo = get_owner_repo(repo_git) @@ -283,11 +351,22 @@ def collect_merge_request_reviewers(mr_ids, repo_git) -> int: logger.info(f"{owner}/{repo} has no gitlab merge request reviewers") def process_mr_reviewers(data, task_name, repo_id, logger, augur_db): + """ + Retrieve only the needed data for mr Reviewer data from the api response + + Arguments: + data: List of dictionaries of mr Reviewer data + repo_id: augur id of the repo + logger: logging object + augur_db: sqlalchemy db object + """ - tool_source = "Mr Reviewr Task" + tool_source = "Mr Reviewer Task" tool_version = "2.0" data_source = "Gitlab API" + logger.info(f"Running {task_name}...") + # create mapping from mr number to pull request id of current mrs mr_number_to_id_map = {} mrs = augur_db.session.query(PullRequest).filter(PullRequest.repo_id == repo_id).all() @@ -299,7 +378,7 @@ def process_mr_reviewers(data, task_name, repo_id, logger, augur_db): pull_request_id = mr_number_to_id_map[id] - reviewers = extract_needed_mr_reviewer_data(values, pull_request_id, repo_id, tool_source, tool_version, data_source) + reviewers = extract_needed_mr_reviewer_data(values, pull_request_id, tool_source, tool_version, data_source) all_reviewers += reviewers @@ -311,6 +390,13 @@ def process_mr_reviewers(data, task_name, repo_id, logger, augur_db): @celery.task(base=AugurCoreRepoCollectionTask) def collect_merge_request_commits(mr_ids, repo_git) -> int: + """ + Retrieve and parse mr commits for the desired repo + + Arguments: + mr_ids: ids of mrs to get commits for + repo_git: the repo url string + """ owner, repo = get_owner_repo(repo_git) @@ -334,6 +420,16 @@ def collect_merge_request_commits(mr_ids, repo_git) -> int: def process_mr_commits(data, task_name, repo_id, logger, augur_db): + """ + Retrieve only the needed data for mr commits from the api response + + Arguments: + data: List of dictionaries of mr commit data + task_name: name of the task as well as the repo being processed + repo_id: augur id of the repo + logger: logging object + augur_db: sqlalchemy db object + """ tool_source = "Mr Commit Task" tool_version = "2.0" @@ -363,6 +459,13 @@ def process_mr_commits(data, task_name, repo_id, logger, augur_db): @celery.task(base=AugurCoreRepoCollectionTask) def collect_merge_request_files(mr_ids, repo_git) -> int: + """ + Retrieve and parse gitlab events for the desired repo + + Arguments: + mr_ids: the ids of mrs to get files for. + repo_git: the repo url string + """ owner, repo = get_owner_repo(repo_git) @@ -409,6 +512,19 @@ def process_mr_files(data, task_name, repo_id, logger, augur_db): def retrieve_merge_request_data(ids, url, name, owner, repo, key_auth, logger, response_type): + """ + Retrieve specific mr data from the GitLab api. + + Arguments: + ids: mr ids to paginate info for + url: endpoint to paginate or hit + name: name of data to collect + owner: owner of the repo + repo: repo name + key_auth: key auth cache and rotator object + logger: loggin object + response_type: type of data to get from the api + """ all_data = {} mr_count = len(ids) @@ -437,7 +553,7 @@ def retrieve_merge_request_data(ids, url, name, owner, repo, key_auth, logger, r else: all_data[id] = page_data else: - raise Exception(f"Unexpected reponse type: {response_type}") + raise Exception(f"Unexpected response type: {response_type}") index += 1 diff --git a/augur/tasks/util/worker_util.py b/augur/tasks/util/worker_util.py index 6380ed22b0..84c177724b 100644 --- a/augur/tasks/util/worker_util.py +++ b/augur/tasks/util/worker_util.py @@ -138,7 +138,7 @@ def parse_json_from_subprocess_call(logger, subprocess_arr, cwd=None): try: required_output = json.loads(output) except json.decoder.JSONDecodeError as e: - session.logger.error(f"Could not parse required output! \n output: {output} \n Error: {e}") + logger.error(f"Could not parse required output! \n output: {output} \n Error: {e}") raise e return required_output diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index a957609265..9c09a4a2f8 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -1,5 +1,5 @@ #SPDX-License-Identifier: MIT -FROM python:3.8.11-slim-buster +FROM python:3.10-slim-bullseye LABEL maintainer="outdoors@acm.org" LABEL version="0.62.0" @@ -13,7 +13,9 @@ RUN set -x \ bash \ curl \ gcc \ - python3-pip \ + musl-dev \ + python3-dev \ + python3-distutils \ wget \ postgresql-client \ && rm -rf /var/lib/apt/lists/* @@ -32,6 +34,17 @@ COPY ./scripts/ scripts/ RUN python3 -m venv /opt/venv RUN set -x \ + && /opt/venv/bin/pip install --upgrade pip + +RUN set -x \ + && /opt/venv/bin/pip install wheel + +RUN set -x \ + && /opt/venv/bin/pip install . + +RUN set -x \ + && /opt/venv/bin/pip install --upgrade pip \ + && /opt/venv/bin/pip install wheel \ && /opt/venv/bin/pip install . RUN ./scripts/docker/install-workers-deps.sh diff --git a/docker/database/Dockerfile b/docker/database/Dockerfile index 610df5b278..1670599754 100644 --- a/docker/database/Dockerfile +++ b/docker/database/Dockerfile @@ -1,5 +1,5 @@ #SPDX-License-Identifier: MIT -FROM postgres:12 +FROM postgres:14 LABEL maintainer="outdoors@acm.org" LABEL version="0.62.0" diff --git a/docs/source/rest-api/spec.yml b/docs/source/rest-api/spec.yml index 566e998346..7b969d5803 100644 --- a/docs/source/rest-api/spec.yml +++ b/docs/source/rest-api/spec.yml @@ -96,6 +96,49 @@ paths: type: array tags: - utility + /repos/:id: + get: + description: Get a downloaded repo by its ID in Augur. The schema block below says it is an array, but it is not. + operationId: Get Repo By ID + responses: + '200': + description: OK + schema: + items: + properties: + base64_url: + description: 'Base64 encode of the full URL. Example Z2l0aHViLmNvbS8zc2NhbGUvM3NjYWxlLW9wZXJhdG9yLW1ldGFkYXRh' + type: string + description: + description: 'Repository description. Example: null' + type: string + commits_all_time: + description: 'How many commits have been made to this repository? Example: 24' + type: integer + issues_all_time: + description: 'How many issues have been raised on this repository? Example: 1' + type: integer + pull_requests_all_time: + description: 'How many pull requests have been made to this repository? Example: 7' + type: integer + repo_id: + description: 'Repository ID, should match provided URL parameter. Example: 25551' + type: integer + repo_name: + description: 'Name of the provided repository. Example: 3scale-operator-metadata' + type: string + rg_name: + description: 'Name of the repository group containing this repo. Example: 3scale' + type: string + repo_group_id: + description: 'ID of the repository group containing this repo. Example: 25431' + type: integer + url: + description: 'URL of this repository, sans leading protocol. Example: github.com/3scale/3scale-operator-metadata' + type: string + type: array + tags: + - utility /owner/:owner/repo/:repo: get: description: Get the repo_group_id and repo_id of a particular repo. @@ -284,12 +327,12 @@ paths: tags: - utility #risk endpoints - /metadata/repo_info: - get: - description: 'Returns the metadata about all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the default branch name, repository license file, forks, stars, watchers, and committers. Also includes metadata about current repository issue and pull request status and counts.' + /metadata/repo_info: + get: + description: 'Returns the metadata about all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the default branch name, repository license file, forks, stars, watchers, and committers. Also includes metadata about current repository issue and pull request status and counts.' externalDocs: description: CHAOSS Metric Definition - url: https://github.com/chaoss/wg-risk/blob/main/focus-areas/business-risk.md + url: https://github.com/chaoss/wg-risk/blob/main/focus-areas/business-risk.md operationId: Activity Metadata (Repo) responses: '200': @@ -315,45 +358,45 @@ paths: fork_count: description: 'Example: 554' type: integer - watchers_count: + watchers_count: description: 'Example: 424' type: integer - stars_count: + stars_count: description: 'Example: 443' type: integer - commits_count: + commits_count: description: '4434' - type: integer - committers_count: + type: integer + committers_count: description: 'Example: 42' type: integer - open_issues: + open_issues: description: 'Example: 7' - type: integer - issues_count: + type: integer + issues_count: description: 'Example: 23332' type: integer - issues_closed: + issues_closed: description: 'Example: 23322' type: integer - pull_request_count: + pull_request_count: description: 'Example: 19445' type: integer - pull_requests_open: + pull_requests_open: description: 'Example: 10' type: integer - pull_requests_closed: + pull_requests_closed: description: 'Example: 19435' type: integer - pull_requests_merged: + pull_requests_merged: description: 'Example: 17473' type: integer type: array tags: - risk - /metadata/contributions_count: - get: - description: 'Returns a list of repositories contributed to by all the contributors in an Augur Instance: INCLUDING all repositories on a platform, *not* merely those repositories in the Augur Instance. Numerical totals represent total CONTRIBUTIONS.' + /metadata/contributions_count: + get: + description: 'Returns a list of repositories contributed to by all the contributors in an Augur Instance: INCLUDING all repositories on a platform, *not* merely those repositories in the Augur Instance. Numerical totals represent total CONTRIBUTIONS.' externalDocs: description: CHAOSS Metric Definition url: https://github.com/chaoss/wg-risk/blob/main/focus-areas/business-risk.md @@ -373,9 +416,9 @@ paths: type: array tags: - risk - /metadata/contributors_count: - get: - description: 'Returns a list of repositories contributed to by all the contributors in an Augur Instance: INCLUDING all repositories on a platform, *not* merely those repositories in the Augur Instance. Numerical totals represent total CONTRIBUTORS.' + /metadata/contributors_count: + get: + description: 'Returns a list of repositories contributed to by all the contributors in an Augur Instance: INCLUDING all repositories on a platform, *not* merely those repositories in the Augur Instance. Numerical totals represent total CONTRIBUTORS.' externalDocs: description: CHAOSS Metric Definition url: https://github.com/chaoss/wg-risk/blob/main/focus-areas/business-risk.md @@ -5228,9 +5271,9 @@ paths: type: object tags: - visualizations - /complexity/project_lines: - get: - description: 'Returns project line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of lines in the project repository.' + /complexity/project_lines: + get: + description: 'Returns project line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of lines in the project repository.' operationId: Total Lines (repo) responses: '200': @@ -5256,9 +5299,9 @@ paths: type: array tags: - complexity - /complexity/project_file_complexity: - get: - description: 'Returns project file complexity data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average file complexity of the project repository.' + /complexity/project_file_complexity: + get: + description: 'Returns project file complexity data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average file complexity of the project repository.' operationId: File Complexity (repo) responses: '200': @@ -5284,9 +5327,9 @@ paths: type: array tags: - complexity - /complexity/project_blank_lines: - get: - description: 'Returns project blank line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of blank lines in the project repository.' + /complexity/project_blank_lines: + get: + description: 'Returns project blank line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of blank lines in the project repository.' operationId: Total Blank Lines (repo) responses: '200': @@ -5312,9 +5355,9 @@ paths: type: array tags: - complexity - /complexity/project_comment_lines: - get: - description: 'Returns project comment line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of comment lines in the project repository.' + /complexity/project_comment_lines: + get: + description: 'Returns project comment line data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total and average number of comment lines in the project repository.' operationId: Total Comment Lines (repo) responses: '200': @@ -5340,9 +5383,9 @@ paths: type: array tags: - complexity - /complexity/project_files: - get: - description: 'Returns project file data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total number of files in the project repository.' + /complexity/project_files: + get: + description: 'Returns project file data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the total number of files in the project repository.' operationId: Total Files (repo) responses: '200': @@ -5365,9 +5408,9 @@ paths: type: array tags: - complexity - /complexity/project_languages: - get: - description: 'Returns project language data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the lines and files of a language in a repository.' + /complexity/project_languages: + get: + description: 'Returns project language data for all repositories in an Augur instance, using information from a git platform (GitHub, GitLab, etc.). Each record includes the lines and files of a language in a repository.' operationId: Project Languages (repo) responses: '200': @@ -5401,7 +5444,7 @@ paths: description: 'The number of messages exchanged for a repository group over a specified period.' externalDocs: description: CHAOSS Metric Definition - url: + url: operationId: Repository Messages (Repo Group) parameters: - description: Repository Group ID @@ -5717,4 +5760,4 @@ paths: type: string enum: ["Missing argument"] tags: - - DEI Badging \ No newline at end of file + - DEI Badging diff --git a/setup.py b/setup.py index d2a149d93b..f87214316e 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ "slack==0.0.2", # 0.0.2 "boto3==1.17.57", # 1.24.56 "toml", # 0.10.2 - "mistune==0.8.4", # 2.0.4 + "mistune", # 2.0.4 "pyYaml", # 6.0 "redis==4.3.3", # 4.3.4 "XlsxWriter==1.3.7", # 3.0.3 @@ -94,11 +94,11 @@ "pytest==6.2.5", # 7.1.2 "toml >= 0.10.2", # 0.10.2 "ipdb==0.13.9", # 0.13.9 - "sphinx==4.2.0", # 5.1.1 - "sphinx_rtd_theme==1.0.0", # 1.0.0 - "sphinxcontrib-openapi==0.7.0", # 0.7.0 + "sphinx==7.2.6", #4.2.0", # 5.1.1 + "sphinx_rtd_theme==2.0.0", # 1.0.0 + "sphinxcontrib-openapi==0.8.3", # 0.7.0 "sphinxcontrib-redoc==1.6.0", # 1.6.0 - "docutils==0.17.1" # 0.19 + "docutils==0.20.1" # 0.19 ] }, entry_points={ diff --git a/tests/test_applicaton/test_db/test_models/test_augur_data/test_repo.py b/tests/test_applicaton/test_db/test_models/test_augur_data/test_repo.py index 1c32472f32..dd1ef44b79 100644 --- a/tests/test_applicaton/test_db/test_models/test_augur_data/test_repo.py +++ b/tests/test_applicaton/test_db/test_models/test_augur_data/test_repo.py @@ -77,20 +77,20 @@ def test_insert_repo(test_db_engine): with DatabaseSession(logger, test_db_engine) as session: - assert Repo.insert_github_repo(session, data["repo_urls"][0], data["rg_id"], data["tool_source"]) is not None - assert Repo.insert_github_repo(session, data["repo_urls"][1], data["rg_id"], data["tool_source"]) is not None + assert Repo.insert_github_repo(session, data["repo_urls"][0], data["rg_id"], data["tool_source"], None) is not None + assert Repo.insert_github_repo(session, data["repo_urls"][1], data["rg_id"], data["tool_source"], None) is not None # invalid rg_id - assert Repo.insert_github_repo(session, data["repo_urls"][0], 12, data["tool_source"]) is None + assert Repo.insert_github_repo(session, data["repo_urls"][0], 12, data["tool_source"], None) is None # invalid type for repo url - assert Repo.insert_github_repo(session, 1, data["rg_id"], data["tool_source"]) is None + assert Repo.insert_github_repo(session, 1, data["rg_id"], data["tool_source"], None) is None # invalid type for rg_id - assert Repo.insert_github_repo(session, data["repo_urls"][1], "1", data["tool_source"]) is None + assert Repo.insert_github_repo(session, data["repo_urls"][1], "1", data["tool_source"], None) is None # invalid type for tool_source - assert Repo.insert_github_repo(session, data["repo_urls"][1], data["rg_id"], 52) is None + assert Repo.insert_github_repo(session, data["repo_urls"][1], data["rg_id"], 52, None) is None with test_db_engine.connect() as connection: