feat(release): add github token to changelog script (#12872)

apache · Feb 2, 2021 · 3ef641d · 3ef641d
1 parent 941d924
commit 3ef641d
Show file tree

Hide file tree

Showing 6 changed files with 47 additions and 47 deletions.
diff --git a/RELEASING/README.md b/RELEASING/README.md
@@ -118,6 +118,8 @@ git push upstream $SUPERSET_GITHUB_BRANCH
 Next, update the `CHANGELOG.md` with all the changes that are included in the release.
 Make sure the branch has been pushed to `upstream` to ensure the changelog generator
 can pick up changes since the previous release.
+Change log script requires a github token and will try to use your env var GITHUB_TOKEN.
+you can also pass the token using the parameter `--access_token`.
 
 Example:
 ```bash

diff --git a/RELEASING/changelog.py b/RELEASING/changelog.py
@@ -17,18 +17,22 @@
 # pylint: disable=no-value-for-parameter
 
 import csv as lib_csv
-import json
 import os
 import re
 import sys
 from dataclasses import dataclass
-from time import sleep
 from typing import Any, Dict, Iterator, List, Optional, Union
-from urllib import request
-from urllib.error import HTTPError
 
 import click
 
+try:
+    from github import BadCredentialsException, Github, PullRequest
+except ModuleNotFoundError:
+    print("PyGithub is a required package for this script")
+    exit(1)
+
+SUPERSET_REPO = "apache/superset"
+
 
 @dataclass
 class GitLog:
@@ -60,47 +64,31 @@ class GitChangeLog:
     We want to map a git author to a github login, for that we call github's API
     """
 
-    def __init__(self, version: str, logs: List[GitLog]) -> None:
+    def __init__(
+        self, version: str, logs: List[GitLog], access_token: Optional[str] = None
+    ) -> None:
         self._version = version
         self._logs = logs
         self._github_login_cache: Dict[str, Optional[str]] = {}
         self._wait = 10
+        github_token = access_token or os.environ.get("GITHUB_TOKEN")
+        self._github = Github(github_token)
+        self._superset_repo = ""
 
-    def _wait_github_rate_limit(self) -> None:
-        """
-        Waits for available rate limit slots on the github API
-        """
-        while True:
-            rate_limit_payload = self._fetch_github_rate_limit()
-            if rate_limit_payload["rate"]["remaining"] > 1:
-                break
-            print(".", end="", flush=True)
-            sleep(self._wait)
-        print()
-
-    @staticmethod
-    def _fetch_github_rate_limit() -> Dict[str, Any]:
-        """
-        Fetches current github rate limit info
-        """
-        with request.urlopen("https://api.github.com/rate_limit") as response:
-            payload = json.loads(response.read())
-        return payload
-
-    def _fetch_github_pr(self, pr_number: int) -> Dict[str, Any]:
+    def _fetch_github_pr(self, pr_number: int) -> PullRequest:
         """
         Fetches a github PR info
         """
-        payload = {}
         try:
-            self._wait_github_rate_limit()
-            with request.urlopen(
-                "https://api.github.com/repos/apache/superset/pulls/" f"{pr_number}"
-            ) as response:
-                payload = json.loads(response.read())
-        except HTTPError as ex:
-            print(f"{ex}", flush=True)
-        return payload
+            github_repo = self._github.get_repo(SUPERSET_REPO)
+        except BadCredentialsException as ex:
+            print(
+                f"Bad credentials to github provided"
+                f" use access_token parameter or set GITHUB_TOKEN"
+            )
+            sys.exit(1)
+
+        return github_repo.get_pull(pr_number)
 
     def _get_github_login(self, git_log: GitLog) -> Optional[str]:
         """
@@ -113,7 +101,7 @@ def _get_github_login(self, git_log: GitLog) -> Optional[str]:
         if git_log.pr_number:
             pr_info = self._fetch_github_pr(git_log.pr_number)
             if pr_info:
-                github_login = pr_info["user"]["login"]
+                github_login = pr_info.user.login
             else:
                 github_login = author_name
         # set cache
@@ -131,7 +119,7 @@ def __repr__(self) -> str:
                 github_login = log.author
             result = result + (
                 f"- [#{log.pr_number}]"
-                f"(https://github.com/apache/superset/pull/{log.pr_number}) "
+                f"(https://github.com/{SUPERSET_REPO}/pull/{log.pr_number}) "
                 f"{log.message} (@{github_login})\n"
             )
             print(f"\r {i}/{len(self._logs)}", end="", flush=True)
@@ -141,7 +129,7 @@ def __iter__(self) -> Iterator[Dict[str, Any]]:
         for log in self._logs:
             yield {
                 "pr_number": log.pr_number,
-                "pr_link": f"https://github.com/apache/superset/pull/"
+                "pr_link": f"https://github.com/{SUPERSET_REPO}/pull/"
                 f"{log.pr_number}",
                 "message": log.message,
                 "time": log.time,
@@ -276,13 +264,20 @@ def compare(base_parameters: BaseParameters) -> None:
 @click.option(
     "--csv", help="The csv filename to export the changelog to",
 )
+@click.option(
+    "--access_token",
+    help="The github access token,"
+    " if not provided will try to fetch from GITHUB_TOKEN env var",
+)
 @click.pass_obj
-def change_log(base_parameters: BaseParameters, csv: str) -> None:
+def change_log(base_parameters: BaseParameters, csv: str, access_token: str) -> None:
     """ Outputs a changelog (by PR) """
     previous_logs = base_parameters.previous_logs
     current_logs = base_parameters.current_logs
     previous_diff_logs = previous_logs.diff(current_logs)
-    logs = GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1])
+    logs = GitChangeLog(
+        current_logs.git_ref, previous_diff_logs[::-1], access_token=access_token
+    )
     if csv:
         with open(csv, "w") as csv_file:
             log_items = list(logs)

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -77,7 +77,7 @@ prison==0.1.3             # via flask-appbuilder
 py==1.9.0                 # via retry
 pyarrow==1.0.1            # via apache-superset
 pycparser==2.20           # via cffi
-pyjwt==1.7.1              # via flask-appbuilder, flask-jwt-extended
+pyjwt==1.7.1              # via apache-superset, flask-appbuilder, flask-jwt-extended
 pymeeus==0.3.7            # via convertdate
 pyparsing==2.4.7          # via apache-superset, packaging
 pyrsistent==0.16.1        # via -r requirements/base.in, jsonschema
@@ -92,12 +92,12 @@ redis==3.5.3              # via apache-superset
 retry==0.9.2              # via apache-superset
 selenium==3.141.0         # via apache-superset
 simplejson==3.17.2        # via apache-superset
-six==1.15.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
+six==1.15.0               # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
 slackclient==2.5.0        # via apache-superset
 sqlalchemy-utils==0.36.8  # via apache-superset, flask-appbuilder
 sqlalchemy==1.3.20        # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
 sqlparse==0.3.0           # via apache-superset
-typing-extensions==3.7.4.3  # via aiohttp, yarl
+typing-extensions==3.7.4.3  # via aiohttp, apache-superset, yarl
 urllib3==1.25.11          # via selenium
 vine==1.3.0               # via amqp, celery
 webencodings==0.5.1       # via bleach

diff --git a/requirements/development.in b/requirements/development.in
@@ -24,3 +24,4 @@ pyhive[hive]>=0.6.1
 psycopg2-binary==2.8.5
 tableschema
 thrift>=0.11.0,<1.0.0
+pygithub>=1.54.1,<2.0.0
diff --git a/requirements/development.txt b/requirements/development.txt
@@ -1,4 +1,4 @@
-# SHA1:ae0364cae066a5cb8fb543c4f568bfcdacb6c1b7
+# SHA1:b7181d683bed10ffe4892d2f07bc94a503f46b44
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -11,6 +11,7 @@ boto3==1.16.10            # via tabulator
 botocore==1.19.10         # via boto3, s3transfer
 cached-property==1.5.2    # via tableschema
 certifi==2020.6.20        # via requests
+deprecated==1.2.11        # via pygithub
 et-xmlfile==1.0.1         # via openpyxl
 flask-cors==3.0.9         # via -r requirements/development.in
 future==0.18.2            # via pyhive
@@ -24,8 +25,9 @@ openpyxl==3.0.5           # via tabulator
 pillow==7.2.0             # via -r requirements/development.in
 psycopg2-binary==2.8.5    # via -r requirements/development.in
 pydruid==0.6.1            # via -r requirements/development.in
+pygithub==1.54.1          # via -r requirements/development.in
 pyhive[hive]==0.6.3       # via -r requirements/development.in
-requests==2.24.0          # via pydruid, tableschema, tabulator
+requests==2.24.0          # via pydruid, pygithub, tableschema, tabulator
 rfc3986==1.4.0            # via tableschema
 s3transfer==0.3.3         # via boto3
 sasl==0.2.1               # via pyhive, thrift-sasl
@@ -34,6 +36,7 @@ tabulator==1.52.5         # via tableschema
 thrift-sasl==0.4.2        # via pyhive
 thrift==0.13.0            # via -r requirements/development.in, pyhive, thrift-sasl
 unicodecsv==0.14.1        # via tableschema, tabulator
+wrapt==1.12.1             # via deprecated
 xlrd==1.2.0               # via tabulator
 
 # The following packages are considered to be unsafe in a requirements file:

diff --git a/requirements/testing.txt b/requirements/testing.txt
@@ -40,7 +40,6 @@ traitlets==5.0.5          # via ipython
 typed-ast==1.4.1          # via astroid
 wcwidth==0.2.5            # via prompt-toolkit
 websocket-client==0.57.0  # via docker
-wrapt==1.12.1             # via astroid
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip