From d1ab7e3eda23e6b21117fc4bd33d896c38cb55f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Wed, 17 Nov 2021 18:09:47 +0800
Subject: [PATCH 01/14] fix typing hint errors(pyright) && format with black
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py | 67 ++++++++++++++++++-----------
 1 file changed, 43 insertions(+), 24 deletions(-)

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 0aca03962..1324ee490 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -26,9 +26,7 @@
 from datetime import datetime
 from json import JSONDecodeError
 from subprocess import check_output
-from typing import List
-from typing import Mapping
-from typing import Set
+from typing import Set, List, MutableMapping, Optional
 
 from aiohttp import ClientSession
 import aiohttp
@@ -41,7 +39,7 @@
 @dataclasses.dataclass(frozen=True)
 class Version:
     value: str
-    release_date: datetime = None
+    release_date: Optional[datetime] = None
 
 
 @dataclasses.dataclass
@@ -55,10 +53,10 @@ class GraphQLError(Exception):
 
 
 class VersionAPI:
-    def __init__(self, cache: Mapping[str, Set[Version]] = None):
+    def __init__(self, cache: MutableMapping[str, Set[Version]] = None):
         self.cache = cache or {}
 
-    def get(self, package_name, until=None) -> Set[str]:
+    def get(self, package_name, until=None) -> VersionResponse:
         new_versions = set()
         valid_versions = set()
         for version in self.cache.get(package_name, set()):
@@ -67,7 +65,9 @@ def get(self, package_name, until=None) -> Set[str]:
                 continue
             valid_versions.add(version.value)
 
-        return VersionResponse(valid_versions=valid_versions, newer_versions=new_versions)
+        return VersionResponse(
+            valid_versions=valid_versions, newer_versions=new_versions
+        )
 
     async def load_api(self, pkg_set):
         async with client_session() as session:
@@ -104,7 +104,7 @@ async def fetch(self, pkg, session):
                 response = await session.request(method="GET", url=url)
                 resp_json = await response.json()
                 if resp_json["entries"] == []:
-                    self.cache[pkg] = {}
+                    self.cache[pkg] = set()
                     break
                 for release in resp_json["entries"]:
                     all_versions.add(
@@ -118,8 +118,12 @@ async def fetch(self, pkg, session):
                 else:
                     break
             self.cache[pkg] = all_versions
-        except (ClientResponseError, asyncio.exceptions.TimeoutError, ServerDisconnectedError):
-            self.cache[pkg] = {}
+        except (
+            ClientResponseError,
+            asyncio.exceptions.TimeoutError,
+            ServerDisconnectedError,
+        ):
+            self.cache[pkg] = set()
 
 
 class PypiVersionAPI(VersionAPI):
@@ -242,7 +246,7 @@ async def fetch(self, pkg, session, retry_count=5):
             resp_json = await response.json()
 
             if resp_json.get("error") or not resp_json.get("versions"):
-                self.cache[pkg] = {}
+                self.cache[pkg] = set()
                 return
             for release in resp_json["versions"]:
                 all_versions.add(Version(value=release["version"].replace("0:", "")))
@@ -250,8 +254,12 @@ async def fetch(self, pkg, session, retry_count=5):
             self.cache[pkg] = all_versions
         # TODO : Handle ServerDisconnectedError by using some sort of
         # retry mechanism
-        except (ClientResponseError, asyncio.exceptions.TimeoutError, ServerDisconnectedError):
-            self.cache[pkg] = {}
+        except (
+            ClientResponseError,
+            asyncio.exceptions.TimeoutError,
+            ServerDisconnectedError,
+        ):
+            self.cache[pkg] = set()
 
 
 class MavenVersionAPI(VersionAPI):
@@ -295,10 +303,10 @@ def artifact_url(artifact_comps: List[str]) -> str:
         return endpoint
 
     @staticmethod
-    def extract_versions(xml_response: ET.ElementTree) -> Set[str]:
+    def extract_versions(xml_response: ET.ElementTree) -> Set[Version]:
         all_versions = set()
         for child in xml_response.getroot().iter():
-            if child.tag == "version":
+            if child.tag == "version" and child.text:
                 all_versions.add(Version(child.text))
 
         return all_versions
@@ -321,7 +329,7 @@ def nuget_url(pkg_name: str) -> str:
         return base_url.format(pkg_name)
 
     @staticmethod
-    def extract_versions(resp: dict) -> Set[str]:
+    def extract_versions(resp: dict) -> Set[Version]:
         all_versions = set()
         try:
             for entry_group in resp["items"]:
@@ -329,7 +337,9 @@ def extract_versions(resp: dict) -> Set[str]:
                     all_versions.add(
                         Version(
                             value=entry["catalogEntry"]["version"],
-                            release_date=dateparser.parse(entry["catalogEntry"]["published"]),
+                            release_date=dateparser.parse(
+                                entry["catalogEntry"]["published"]
+                            ),
                         )
                     )
         # FIXME: json response for YamlDotNet.Signed triggers this exception.
@@ -353,7 +363,7 @@ async def fetch(self, pkg, session) -> None:
             self.cache[pkg] = self.extract_versions(resp, pkg)
 
     @staticmethod
-    def composer_url(pkg_name: str) -> str:
+    def composer_url(pkg_name: str) -> Optional[str]:
         try:
             vendor, name = pkg_name.split("/")
         except ValueError:
@@ -362,7 +372,7 @@ def composer_url(pkg_name: str) -> str:
         return f"https://repo.packagist.org/p/{vendor}/{name}.json"
 
     @staticmethod
-    def extract_versions(resp: dict, pkg_name: str) -> Set[str]:
+    def extract_versions(resp: dict, pkg_name: str) -> Set[Version]:
         all_versions = set()
         for version in resp["packages"][pkg_name]:
             if "dev" in version:
@@ -374,7 +384,9 @@ def extract_versions(resp: dict, pkg_name: str) -> Set[str]:
             all_versions.add(
                 Version(
                     value=version.lstrip("v"),
-                    release_date=dateparser.parse(resp["packages"][pkg_name][version]["time"]),
+                    release_date=dateparser.parse(
+                        resp["packages"][pkg_name][version]["time"]
+                    ),
                 )
             )
         return all_versions
@@ -412,7 +424,7 @@ class GitHubTagsAPI(VersionAPI):
         }
     }"""
 
-    def __init__(self, cache: Mapping[str, Set[Version]] = None):
+    def __init__(self, cache: MutableMapping[str, Set[Version]] = None):
         self.gh_token = os.getenv("GH_TOKEN")
         super().__init__(cache=cache)
 
@@ -427,7 +439,10 @@ async def fetch(self, owner_repo: str, session: aiohttp.ClientSession) -> None:
             session.headers["Authorization"] = "token " + self.gh_token
             endpoint = f"https://api.github.com/graphql"
             owner, name = owner_repo.split("/")
-            query = {"query": self.GQL_QUERY, "variables": {"name": name, "owner": owner}}
+            query = {
+                "query": self.GQL_QUERY,
+                "variables": {"name": name, "owner": owner},
+            }
 
             while True:
                 response = await session.post(endpoint, json=query)
@@ -451,7 +466,9 @@ async def fetch(self, owner_repo: str, session: aiohttp.ClientSession) -> None:
                         # probably this only happened for linux. Github cannot even properly display it.
                         # https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+/refs/tags/v2.6.11
                         release_date = None
-                    self.cache[owner_repo].add(Version(value=name, release_date=release_date))
+                    self.cache[owner_repo].add(
+                        Version(value=name, release_date=release_date)
+                    )
 
                 if not refs["pageInfo"]["hasNextPage"]:
                     break
@@ -464,7 +481,9 @@ async def fetch(self, owner_repo: str, session: aiohttp.ClientSession) -> None:
             # this method is however not scalable for larger repo and the api is unresponsive
             # for repo with > 50 tags
             endpoint = f"https://github.com/{owner_repo}"
-            tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
+            tags_xml = check_output(
+                ["svn", "ls", "--xml", f"{endpoint}/tags"], text=True
+            )
             elements = ET.fromstring(tags_xml)
             for entry in elements.iter("entry"):
                 name = entry.find("name").text

From 7d8334bf058ba26d3d89f435178e37c1ef833d03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Wed, 17 Nov 2021 19:22:19 +0800
Subject: [PATCH 02/14] add GoproxyVersionAPI
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py | 69 +++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 1324ee490..161addd0e 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -27,6 +27,7 @@
 from json import JSONDecodeError
 from subprocess import check_output
 from typing import Set, List, MutableMapping, Optional
+from django.utils.dateparse import parse_datetime
 
 from aiohttp import ClientSession
 import aiohttp
@@ -508,3 +509,71 @@ async def fetch(self, pkg, session):
             pass
 
         self.cache[pkg] = versions
+
+
+class GoproxyVersionAPI(VersionAPI):
+
+    package_type = "golang"
+
+    @staticmethod
+    def trim_package_path(pkg: str) -> Optional[str]:
+        """github advisories for golang is using package names(e.g. https://github.com/advisories/GHSA-jp4j-47f9-2vc3), yet goproxy works with module names(see https://golang.org/ref/mod#goproxy-protocol).
+        this method removes the last part of a package path, and returns the remaining as the module name.
+        """
+        # some advisories contains this prefix in package name, e.g. https://github.com/advisories/GHSA-7h6j-2268-fhcm
+        pkg = pkg.removeprefix("https://pkg.go.dev/")
+        parts = pkg.split("/")
+        if len(parts) >= 2:
+            return "/".join(parts[:-1])
+        else:
+            return None
+
+    @staticmethod
+    def escape_path(path: str) -> str:
+        """escpe uppercase in module/version name"""
+        escaped_path = ""
+        for c in path:
+            if c >= "A" and c <= "Z":
+                escaped_path += "!" + chr(ord(c) + ord("a") - ord("A"))
+            else:
+                escaped_path += c
+        return escaped_path
+
+    async def fetch(self, pkg: str, session: ClientSession):
+        # escpe uppercase in module path
+        escaped_pkg = GoproxyVersionAPI.escape_path(pkg)
+        resp_text = None
+        err_pkgs = []
+        while escaped_pkg is not None:
+            url = f"https://proxy.golang.org/{escaped_pkg}/@v/list"
+            try:
+                response = await session.request(method="GET", url=url)
+                resp_text = await response.text()
+            except:
+                err_pkgs.append(escaped_pkg)
+                escaped_pkg = GoproxyVersionAPI.trim_package_path(escaped_pkg)
+                continue
+            break
+        if resp_text is None:
+            print("error fetch versions from goproxy: ", err_pkgs)
+            return
+        versions = set()
+        for version_info in resp_text.split("\n"):
+            v = version_info.split()
+            if len(v) > 0:
+                value = v[0]
+                if len(v) > 1:
+                    release_date = parse_datetime(v[1])
+                else:
+                    escaped_ver = GoproxyVersionAPI.escape_path(value)
+                    try:
+                        response = await session.request(
+                            method="GET",
+                            url=f"https://proxy.golang.org/{escaped_pkg}/@v/{escaped_ver}.info",
+                        )
+                        resp_json = await response.json()
+                        release_date = parse_datetime(resp_json.get("Time", ""))
+                    except:
+                        release_date = None
+                versions.add(Version(value=value, release_date=release_date))
+        self.cache[pkg] = versions

From 91248e9a6ef444d58bbe270c87fdb1a7786efdad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Wed, 17 Nov 2021 20:18:26 +0800
Subject: [PATCH 03/14] github api add GO ecosystem
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/importer_yielder.py | 2 +-
 vulnerabilities/importers/github.py | 6 +++++-
 vulnerabilities/package_managers.py | 6 +++++-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py
index 567755c3c..04f921230 100644
--- a/vulnerabilities/importer_yielder.py
+++ b/vulnerabilities/importer_yielder.py
@@ -163,7 +163,7 @@
         "data_source": "GitHubAPIDataSource",
         "data_source_cfg": {
             "endpoint": "https://api.github.com/graphql",
-            "ecosystems": ["MAVEN", "NUGET", "COMPOSER", "PIP", "RUBYGEMS"],
+            "ecosystems": ["MAVEN", "NUGET", "COMPOSER", "PIP", "RUBYGEMS", "GO"],
         },
     },
     {
diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py
index b1cee1fda..b6c8ca19b 100644
--- a/vulnerabilities/importers/github.py
+++ b/vulnerabilities/importers/github.py
@@ -44,6 +44,7 @@
 from vulnerabilities.package_managers import NugetVersionAPI
 from vulnerabilities.package_managers import ComposerVersionAPI
 from vulnerabilities.package_managers import PypiVersionAPI
+from vulnerabilities.package_managers import GoproxyVersionAPI
 from vulnerabilities.package_managers import RubyVersionAPI
 from vulnerabilities.severity_systems import scoring_systems
 from vulnerabilities.helpers import nearest_patched_package
@@ -206,6 +207,7 @@ def set_version_api(self, ecosystem: str) -> None:
             "COMPOSER": ComposerVersionAPI,
             "PIP": PypiVersionAPI,
             "RUBYGEMS": RubyVersionAPI,
+            "GO": GoproxyVersionAPI,
         }
         versioner = versioners.get(ecosystem)
         if versioner:
@@ -229,7 +231,7 @@ def process_name(ecosystem: str, pkg_name: str) -> Optional[Tuple[Optional[str],
                 return None
             return vendor, name
 
-        if ecosystem == "NUGET" or ecosystem == "PIP" or ecosystem == "RUBYGEMS":
+        if ecosystem == "NUGET" or ecosystem == "PIP" or ecosystem == "RUBYGEMS" or ecosystem == "GO":
             return None, pkg_name
 
     @staticmethod
@@ -265,6 +267,8 @@ def process_response(self) -> List[Advisory]:
                     unaffected_purls = []
                     if self.process_name(ecosystem, name):
                         ns, pkg_name = self.process_name(ecosystem, name)
+                        if hasattr(self.version_api, "pkg_mappings"):
+                            pkg_name = self.version_api.pkg_mappings.get(name, pkg_name)
                         aff_range = adv["node"]["vulnerableVersionRange"]
                         aff_vers, unaff_vers = self.categorize_versions(
                             self.version_api.package_type,
diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 161addd0e..e45421a79 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -514,6 +514,7 @@ async def fetch(self, pkg, session):
 class GoproxyVersionAPI(VersionAPI):
 
     package_type = "golang"
+    pkg_mappings = {}
 
     @staticmethod
     def trim_package_path(pkg: str) -> Optional[str]:
@@ -542,6 +543,7 @@ def escape_path(path: str) -> str:
     async def fetch(self, pkg: str, session: ClientSession):
         # escpe uppercase in module path
         escaped_pkg = GoproxyVersionAPI.escape_path(pkg)
+        trimmed_pkg = pkg
         resp_text = None
         err_pkgs = []
         while escaped_pkg is not None:
@@ -550,13 +552,15 @@ async def fetch(self, pkg: str, session: ClientSession):
                 response = await session.request(method="GET", url=url)
                 resp_text = await response.text()
             except:
-                err_pkgs.append(escaped_pkg)
+                err_pkgs.append(trimmed_pkg)
                 escaped_pkg = GoproxyVersionAPI.trim_package_path(escaped_pkg)
+                trimmed_pkg = GoproxyVersionAPI.trim_package_path(trimmed_pkg) or ""
                 continue
             break
         if resp_text is None:
             print("error fetch versions from goproxy: ", err_pkgs)
             return
+        self.pkg_mappings[pkg] = trimmed_pkg
         versions = set()
         for version_info in resp_text.split("\n"):
             v = version_info.split()

From 798cb654cdb1a3784963d8100e1ee4e5c1d8f4bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Tue, 30 Nov 2021 10:39:52 +0800
Subject: [PATCH 04/14] code refactor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py | 70 +++++++++++++++++------------
 1 file changed, 41 insertions(+), 29 deletions(-)

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index e45421a79..76679bc12 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -33,6 +33,7 @@
 import aiohttp
 from aiohttp.client_exceptions import ClientResponseError
 from aiohttp.client_exceptions import ServerDisconnectedError
+from aiohttp.web_exceptions import HTTPGone
 from bs4 import BeautifulSoup
 from dateutil import parser as dateparser
 
@@ -517,13 +518,14 @@ class GoproxyVersionAPI(VersionAPI):
     pkg_mappings = {}
 
     @staticmethod
-    def trim_package_path(pkg: str) -> Optional[str]:
+    def trim_url_path(url_path: str) -> Optional[str]:
         """github advisories for golang is using package names(e.g. https://github.com/advisories/GHSA-jp4j-47f9-2vc3), yet goproxy works with module names(see https://golang.org/ref/mod#goproxy-protocol).
         this method removes the last part of a package path, and returns the remaining as the module name.
         """
         # some advisories contains this prefix in package name, e.g. https://github.com/advisories/GHSA-7h6j-2268-fhcm
-        pkg = pkg.removeprefix("https://pkg.go.dev/")
-        parts = pkg.split("/")
+        if url_path.startswith("https://pkg.go.dev/"):
+            url_path = url_path.removeprefix("https://pkg.go.dev/")
+        parts = url_path.split("/")
         if len(parts) >= 2:
             return "/".join(parts[:-1])
         else:
@@ -531,53 +533,63 @@ def trim_package_path(pkg: str) -> Optional[str]:
 
     @staticmethod
     def escape_path(path: str) -> str:
-        """escpe uppercase in module/version name"""
+        """escape uppercase in module/version name"""
         escaped_path = ""
         for c in path:
             if c >= "A" and c <= "Z":
+                # replace uppercase with !lowercase
                 escaped_path += "!" + chr(ord(c) + ord("a") - ord("A"))
             else:
                 escaped_path += c
         return escaped_path
 
+    @staticmethod
+    async def parse_version_info(
+        version_info: str, escaped_pkg: str, session: ClientSession
+    ) -> Optional[Version]:
+        v = version_info.split()
+        if len(v) > 0:
+            value = v[0]
+            if len(v) > 1:
+                release_date = parse_datetime(v[1])
+            else:
+                escaped_ver = GoproxyVersionAPI.escape_path(value)
+                try:
+                    response = await session.request(
+                        method="GET",
+                        url=f"https://proxy.golang.org/{escaped_pkg}/@v/{escaped_ver}.info",
+                    )
+                    resp_json = await response.json()
+                    release_date = parse_datetime(resp_json.get("Time", ""))
+                except:
+                    release_date = None
+            return Version(value=value, release_date=release_date)
+        return None
+
     async def fetch(self, pkg: str, session: ClientSession):
-        # escpe uppercase in module path
+        # escape uppercase in module path
         escaped_pkg = GoproxyVersionAPI.escape_path(pkg)
         trimmed_pkg = pkg
         resp_text = None
-        err_pkgs = []
         while escaped_pkg is not None:
             url = f"https://proxy.golang.org/{escaped_pkg}/@v/list"
             try:
                 response = await session.request(method="GET", url=url)
                 resp_text = await response.text()
-            except:
-                err_pkgs.append(trimmed_pkg)
-                escaped_pkg = GoproxyVersionAPI.trim_package_path(escaped_pkg)
-                trimmed_pkg = GoproxyVersionAPI.trim_package_path(trimmed_pkg) or ""
+            except HTTPGone:
+                escaped_pkg = GoproxyVersionAPI.trim_url_path(escaped_pkg)
+                trimmed_pkg = GoproxyVersionAPI.trim_url_path(trimmed_pkg) or ""
                 continue
             break
-        if resp_text is None:
-            print("error fetch versions from goproxy: ", err_pkgs)
+        if resp_text is None or escaped_pkg is None or trimmed_pkg is None:
+            print("error fetch versions from goproxy: ", pkg)
             return
         self.pkg_mappings[pkg] = trimmed_pkg
         versions = set()
         for version_info in resp_text.split("\n"):
-            v = version_info.split()
-            if len(v) > 0:
-                value = v[0]
-                if len(v) > 1:
-                    release_date = parse_datetime(v[1])
-                else:
-                    escaped_ver = GoproxyVersionAPI.escape_path(value)
-                    try:
-                        response = await session.request(
-                            method="GET",
-                            url=f"https://proxy.golang.org/{escaped_pkg}/@v/{escaped_ver}.info",
-                        )
-                        resp_json = await response.json()
-                        release_date = parse_datetime(resp_json.get("Time", ""))
-                    except:
-                        release_date = None
-                versions.add(Version(value=value, release_date=release_date))
+            version = await GoproxyVersionAPI.parse_version_info(
+                version_info, escaped_pkg, session
+            )
+            if version is not None:
+                versions.add(version)
         self.cache[pkg] = versions

From 853a644fa87882c965911ea062ef06b3932b8b6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Wed, 15 Dec 2021 11:19:23 +0800
Subject: [PATCH 05/14] add comments and examples
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/importers/github.py |  4 ++--
 vulnerabilities/package_managers.py | 11 ++++++-----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py
index b6c8ca19b..7269ad6e5 100644
--- a/vulnerabilities/importers/github.py
+++ b/vulnerabilities/importers/github.py
@@ -267,8 +267,8 @@ def process_response(self) -> List[Advisory]:
                     unaffected_purls = []
                     if self.process_name(ecosystem, name):
                         ns, pkg_name = self.process_name(ecosystem, name)
-                        if hasattr(self.version_api, "pkg_mappings"):
-                            pkg_name = self.version_api.pkg_mappings.get(name, pkg_name)
+                        if hasattr(self.version_api, "module_name_by_package_name"):
+                            pkg_name = self.version_api.module_name_by_package_name.get(name, pkg_name)
                         aff_range = adv["node"]["vulnerableVersionRange"]
                         aff_vers, unaff_vers = self.categorize_versions(
                             self.version_api.package_type,
diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 76679bc12..18671b624 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -515,12 +515,12 @@ async def fetch(self, pkg, session):
 class GoproxyVersionAPI(VersionAPI):
 
     package_type = "golang"
-    pkg_mappings = {}
+    module_name_by_package_name = {}
 
     @staticmethod
     def trim_url_path(url_path: str) -> Optional[str]:
         """github advisories for golang is using package names(e.g. https://github.com/advisories/GHSA-jp4j-47f9-2vc3), yet goproxy works with module names(see https://golang.org/ref/mod#goproxy-protocol).
-        this method removes the last part of a package path, and returns the remaining as the module name.
+        this method removes the last part of a package path, and returns the remaining as the module name. For example: trim_url_path("https://github.com/xx/a/b") returns "https://github.com/xx/a"
         """
         # some advisories contains this prefix in package name, e.g. https://github.com/advisories/GHSA-7h6j-2268-fhcm
         if url_path.startswith("https://pkg.go.dev/"):
@@ -533,7 +533,7 @@ def trim_url_path(url_path: str) -> Optional[str]:
 
     @staticmethod
     def escape_path(path: str) -> str:
-        """escape uppercase in module/version name"""
+        """escape uppercase in module/version name. For example: escape_path("github.com/FerretDB/FerretDB") returns "github.com/!ferret!d!b/!ferret!d!b" """
         escaped_path = ""
         for c in path:
             if c >= "A" and c <= "Z":
@@ -571,6 +571,7 @@ async def fetch(self, pkg: str, session: ClientSession):
         escaped_pkg = GoproxyVersionAPI.escape_path(pkg)
         trimmed_pkg = pkg
         resp_text = None
+        # resolve module name from package name, see https://go.dev/ref/mod#resolve-pkg-mod
         while escaped_pkg is not None:
             url = f"https://proxy.golang.org/{escaped_pkg}/@v/list"
             try:
@@ -582,9 +583,9 @@ async def fetch(self, pkg: str, session: ClientSession):
                 continue
             break
         if resp_text is None or escaped_pkg is None or trimmed_pkg is None:
-            print("error fetch versions from goproxy: ", pkg)
+            print(f"error while fetching versions for {pkg} from goproxy")
             return
-        self.pkg_mappings[pkg] = trimmed_pkg
+        self.module_name_by_package_name[pkg] = trimmed_pkg
         versions = set()
         for version_info in resp_text.split("\n"):
             version = await GoproxyVersionAPI.parse_version_info(

From 5f50caf7d76e09abe784591ee77c7dec0dbb188f Mon Sep 17 00:00:00 2001
From: sify21 <sify21@163.com>
Date: Mon, 7 Feb 2022 11:14:58 +0800
Subject: [PATCH 06/14] Update vulnerabilities/importers/github.py

Co-authored-by: Philippe Ombredanne <pombredanne@gmail.com>
---
 vulnerabilities/importers/github.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py
index 7269ad6e5..78c4b1ab0 100644
--- a/vulnerabilities/importers/github.py
+++ b/vulnerabilities/importers/github.py
@@ -231,7 +231,7 @@ def process_name(ecosystem: str, pkg_name: str) -> Optional[Tuple[Optional[str],
                 return None
             return vendor, name
 
-        if ecosystem == "NUGET" or ecosystem == "PIP" or ecosystem == "RUBYGEMS" or ecosystem == "GO":
+        if ecosystem in ("NUGET", "PIP", "RUBYGEMS", "GO"):
             return None, pkg_name
 
     @staticmethod

From 959006a757d56b111f7427a044025af678a09ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Mon, 7 Feb 2022 11:21:39 +0800
Subject: [PATCH 07/14] wrap docstring to shorter length
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 18671b624..8332b952d 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -519,8 +519,14 @@ class GoproxyVersionAPI(VersionAPI):
 
     @staticmethod
     def trim_url_path(url_path: str) -> Optional[str]:
-        """github advisories for golang is using package names(e.g. https://github.com/advisories/GHSA-jp4j-47f9-2vc3), yet goproxy works with module names(see https://golang.org/ref/mod#goproxy-protocol).
-        this method removes the last part of a package path, and returns the remaining as the module name. For example: trim_url_path("https://github.com/xx/a/b") returns "https://github.com/xx/a"
+        """
+        github advisories for golang is using package names(e.g. 
+        https://github.com/advisories/GHSA-jp4j-47f9-2vc3), 
+        yet goproxy works with module names(see 
+        https://golang.org/ref/mod#goproxy-protocol).
+        this method removes the last part of a package path, and returns the 
+        remaining as the module name. For example: trim_url_path(
+        "https://github.com/xx/a/b") returns "https://github.com/xx/a"
         """
         # some advisories contains this prefix in package name, e.g. https://github.com/advisories/GHSA-7h6j-2268-fhcm
         if url_path.startswith("https://pkg.go.dev/"):
@@ -533,7 +539,11 @@ def trim_url_path(url_path: str) -> Optional[str]:
 
     @staticmethod
     def escape_path(path: str) -> str:
-        """escape uppercase in module/version name. For example: escape_path("github.com/FerretDB/FerretDB") returns "github.com/!ferret!d!b/!ferret!d!b" """
+        """
+        escape uppercase in module/version name. For example: 
+        escape_path("github.com/FerretDB/FerretDB") returns 
+        "github.com/!ferret!d!b/!ferret!d!b" 
+        """
         escaped_path = ""
         for c in path:
             if c >= "A" and c <= "Z":

From 7cb6540061ce05d98ffbdfe9fa50c767490ce56f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Mon, 7 Feb 2022 16:29:41 +0800
Subject: [PATCH 08/14] add unittest
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py           |  4 +-
 .../test_data/goproxy_api/ferretdb_versions   |  5 ++
 .../tests/test_data/goproxy_api/version_info  |  1 +
 .../tests/test_package_managers.py            | 60 ++++++++++++++++++-
 4 files changed, 66 insertions(+), 4 deletions(-)
 create mode 100644 vulnerabilities/tests/test_data/goproxy_api/ferretdb_versions
 create mode 100644 vulnerabilities/tests/test_data/goproxy_api/version_info

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 8332b952d..9bdf670c2 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -540,9 +540,7 @@ def trim_url_path(url_path: str) -> Optional[str]:
     @staticmethod
     def escape_path(path: str) -> str:
         """
-        escape uppercase in module/version name. For example: 
-        escape_path("github.com/FerretDB/FerretDB") returns 
-        "github.com/!ferret!d!b/!ferret!d!b" 
+        escape uppercase in module/version name.
         """
         escaped_path = ""
         for c in path:
diff --git a/vulnerabilities/tests/test_data/goproxy_api/ferretdb_versions b/vulnerabilities/tests/test_data/goproxy_api/ferretdb_versions
new file mode 100644
index 000000000..70230bc4b
--- /dev/null
+++ b/vulnerabilities/tests/test_data/goproxy_api/ferretdb_versions
@@ -0,0 +1,5 @@
+v0.0.1
+v0.0.5
+v0.0.3
+v0.0.4
+v0.0.2
diff --git a/vulnerabilities/tests/test_data/goproxy_api/version_info b/vulnerabilities/tests/test_data/goproxy_api/version_info
new file mode 100644
index 000000000..7774c4a3f
--- /dev/null
+++ b/vulnerabilities/tests/test_data/goproxy_api/version_info
@@ -0,0 +1 @@
+{"Version":"v0.0.5","Time":"2022-01-04T13:54:01Z"}
\ No newline at end of file
diff --git a/vulnerabilities/tests/test_package_managers.py b/vulnerabilities/tests/test_package_managers.py
index 2e74f7c54..d9e10bfd2 100644
--- a/vulnerabilities/tests/test_package_managers.py
+++ b/vulnerabilities/tests/test_package_managers.py
@@ -28,12 +28,13 @@
 import xml.etree.ElementTree as ET
 from datetime import datetime
 from aiohttp.client import ClientSession
-from dateutil.tz import tzlocal
+from dateutil.tz import tzlocal, tzutc
 from pytz import UTC
 from unittest import TestCase
 from unittest.mock import AsyncMock
 
 from vulnerabilities.package_managers import ComposerVersionAPI
+from vulnerabilities.package_managers import GoproxyVersionAPI
 from vulnerabilities.package_managers import MavenVersionAPI
 from vulnerabilities.package_managers import NugetVersionAPI
 from vulnerabilities.package_managers import GitHubTagsAPI
@@ -53,6 +54,7 @@ async def request(self, *args, **kwargs):
         mock_response = AsyncMock()
         mock_response.json = self.json
         mock_response.read = self.read
+        mock_response.text = self.text
         return mock_response
 
     def get(self, *args, **kwargs):
@@ -69,6 +71,9 @@ async def json(self):
     async def read(self):
         return self.return_val
 
+    async def text(self):
+        return self.return_val
+
 
 class RecordedClientSession:
     def __init__(self, test_id, regen=False):
@@ -448,6 +453,59 @@ def test_fetch(self):
         assert self.version_api.get("org.apache:kafka") == VersionResponse(valid_versions=expected)
 
 
+class TestGoproxyVersionAPI(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.version_api = GoproxyVersionAPI()
+        with open(os.path.join(TEST_DATA, "goproxy_api", "ferretdb_versions")) as f:
+            cls.vlist = f.read()
+        with open(os.path.join(TEST_DATA, "goproxy_api", "version_info")) as f:
+            cls.vinfo = json.load(f)
+        cls.expected_versions = {
+            Version(value="v0.0.1"),
+            Version(value="v0.0.2"),
+            Version(value="v0.0.3"),
+            Version(value="v0.0.4"),
+            Version(value="v0.0.5"),
+        }
+
+    def test_trim_url_path(self):
+        url1 = "https://pkg.go.dev/github.com/containous/traefik/v2"
+        url2 = "github.com/FerretDB/FerretDB/cmd/ferretdb"
+        url3 = self.version_api.trim_url_path(url2)
+        assert "github.com/containous/traefik" == self.version_api.trim_url_path(url1)
+        assert "github.com/FerretDB/FerretDB/cmd" == url3
+        assert "github.com/FerretDB/FerretDB" == self.version_api.trim_url_path(url3)
+
+    def test_escape_path(self):
+        path = "github.com/FerretDB/FerretDB"
+        assert "github.com/!ferret!d!b/!ferret!d!b" == self.version_api.escape_path(
+            path
+        )
+
+    def test_parse_version_info(self):
+        client_session = MockClientSession(self.vinfo)
+        assert asyncio.run(
+            self.version_api.parse_version_info(
+                "v0.0.5", "github.com/!ferret!d!b/!ferret!d!b", client_session
+            )
+        ) == Version(
+            value="v0.0.5",
+            release_date=datetime(2022, 1, 4, 13, 54, 1, tzinfo=tzutc()),
+        )
+
+    def test_fetch(self):
+        assert self.version_api.get("github.com/FerretDB/FerretDB") == VersionResponse()
+        client_session = MockClientSession(self.vlist)
+        asyncio.run(
+            self.version_api.fetch("github.com/FerretDB/FerretDB", client_session)
+        )
+        assert (
+            self.version_api.cache["github.com/FerretDB/FerretDB"]
+            == self.expected_versions
+        )
+
+
 class TestNugetVersionAPI(TestCase):
     @classmethod
     def setUpClass(cls):

From b7b7fde132c52c35ef34102db2e89783a462cb1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Tue, 8 Feb 2022 17:01:38 +0800
Subject: [PATCH 09/14] reformat with black
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/importers/github.py           |  4 ++-
 vulnerabilities/package_managers.py           | 32 ++++++-------------
 .../tests/test_package_managers.py            | 13 ++------
 3 files changed, 16 insertions(+), 33 deletions(-)

diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/importers/github.py
index 78c4b1ab0..03aec2e2f 100644
--- a/vulnerabilities/importers/github.py
+++ b/vulnerabilities/importers/github.py
@@ -268,7 +268,9 @@ def process_response(self) -> List[Advisory]:
                     if self.process_name(ecosystem, name):
                         ns, pkg_name = self.process_name(ecosystem, name)
                         if hasattr(self.version_api, "module_name_by_package_name"):
-                            pkg_name = self.version_api.module_name_by_package_name.get(name, pkg_name)
+                            pkg_name = self.version_api.module_name_by_package_name.get(
+                                name, pkg_name
+                            )
                         aff_range = adv["node"]["vulnerableVersionRange"]
                         aff_vers, unaff_vers = self.categorize_versions(
                             self.version_api.package_type,
diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 9bdf670c2..ca9e88163 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -67,9 +67,7 @@ def get(self, package_name, until=None) -> VersionResponse:
                 continue
             valid_versions.add(version.value)
 
-        return VersionResponse(
-            valid_versions=valid_versions, newer_versions=new_versions
-        )
+        return VersionResponse(valid_versions=valid_versions, newer_versions=new_versions)
 
     async def load_api(self, pkg_set):
         async with client_session() as session:
@@ -339,9 +337,7 @@ def extract_versions(resp: dict) -> Set[Version]:
                     all_versions.add(
                         Version(
                             value=entry["catalogEntry"]["version"],
-                            release_date=dateparser.parse(
-                                entry["catalogEntry"]["published"]
-                            ),
+                            release_date=dateparser.parse(entry["catalogEntry"]["published"]),
                         )
                     )
         # FIXME: json response for YamlDotNet.Signed triggers this exception.
@@ -386,9 +382,7 @@ def extract_versions(resp: dict, pkg_name: str) -> Set[Version]:
             all_versions.add(
                 Version(
                     value=version.lstrip("v"),
-                    release_date=dateparser.parse(
-                        resp["packages"][pkg_name][version]["time"]
-                    ),
+                    release_date=dateparser.parse(resp["packages"][pkg_name][version]["time"]),
                 )
             )
         return all_versions
@@ -468,9 +462,7 @@ async def fetch(self, owner_repo: str, session: aiohttp.ClientSession) -> None:
                         # probably this only happened for linux. Github cannot even properly display it.
                         # https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+/refs/tags/v2.6.11
                         release_date = None
-                    self.cache[owner_repo].add(
-                        Version(value=name, release_date=release_date)
-                    )
+                    self.cache[owner_repo].add(Version(value=name, release_date=release_date))
 
                 if not refs["pageInfo"]["hasNextPage"]:
                     break
@@ -483,9 +475,7 @@ async def fetch(self, owner_repo: str, session: aiohttp.ClientSession) -> None:
             # this method is however not scalable for larger repo and the api is unresponsive
             # for repo with > 50 tags
             endpoint = f"https://github.com/{owner_repo}"
-            tags_xml = check_output(
-                ["svn", "ls", "--xml", f"{endpoint}/tags"], text=True
-            )
+            tags_xml = check_output(["svn", "ls", "--xml", f"{endpoint}/tags"], text=True)
             elements = ET.fromstring(tags_xml)
             for entry in elements.iter("entry"):
                 name = entry.find("name").text
@@ -520,11 +510,11 @@ class GoproxyVersionAPI(VersionAPI):
     @staticmethod
     def trim_url_path(url_path: str) -> Optional[str]:
         """
-        github advisories for golang is using package names(e.g. 
-        https://github.com/advisories/GHSA-jp4j-47f9-2vc3), 
-        yet goproxy works with module names(see 
+        github advisories for golang is using package names(e.g.
+        https://github.com/advisories/GHSA-jp4j-47f9-2vc3),
+        yet goproxy works with module names(see
         https://golang.org/ref/mod#goproxy-protocol).
-        this method removes the last part of a package path, and returns the 
+        this method removes the last part of a package path, and returns the
         remaining as the module name. For example: trim_url_path(
         "https://github.com/xx/a/b") returns "https://github.com/xx/a"
         """
@@ -596,9 +586,7 @@ async def fetch(self, pkg: str, session: ClientSession):
         self.module_name_by_package_name[pkg] = trimmed_pkg
         versions = set()
         for version_info in resp_text.split("\n"):
-            version = await GoproxyVersionAPI.parse_version_info(
-                version_info, escaped_pkg, session
-            )
+            version = await GoproxyVersionAPI.parse_version_info(version_info, escaped_pkg, session)
             if version is not None:
                 versions.add(version)
         self.cache[pkg] = versions
diff --git a/vulnerabilities/tests/test_package_managers.py b/vulnerabilities/tests/test_package_managers.py
index d9e10bfd2..626337c64 100644
--- a/vulnerabilities/tests/test_package_managers.py
+++ b/vulnerabilities/tests/test_package_managers.py
@@ -479,9 +479,7 @@ def test_trim_url_path(self):
 
     def test_escape_path(self):
         path = "github.com/FerretDB/FerretDB"
-        assert "github.com/!ferret!d!b/!ferret!d!b" == self.version_api.escape_path(
-            path
-        )
+        assert "github.com/!ferret!d!b/!ferret!d!b" == self.version_api.escape_path(path)
 
     def test_parse_version_info(self):
         client_session = MockClientSession(self.vinfo)
@@ -497,13 +495,8 @@ def test_parse_version_info(self):
     def test_fetch(self):
         assert self.version_api.get("github.com/FerretDB/FerretDB") == VersionResponse()
         client_session = MockClientSession(self.vlist)
-        asyncio.run(
-            self.version_api.fetch("github.com/FerretDB/FerretDB", client_session)
-        )
-        assert (
-            self.version_api.cache["github.com/FerretDB/FerretDB"]
-            == self.expected_versions
-        )
+        asyncio.run(self.version_api.fetch("github.com/FerretDB/FerretDB", client_session))
+        assert self.version_api.cache["github.com/FerretDB/FerretDB"] == self.expected_versions
 
 
 class TestNugetVersionAPI(TestCase):

From 3abe12d647f866b724d03c59ee518e42abaef438 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Wed, 9 Feb 2022 14:30:43 +0800
Subject: [PATCH 10/14] inline test setups
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 .../tests/test_package_managers.py            | 46 +++++++++----------
 1 file changed, 21 insertions(+), 25 deletions(-)

diff --git a/vulnerabilities/tests/test_package_managers.py b/vulnerabilities/tests/test_package_managers.py
index 626337c64..366ddee49 100644
--- a/vulnerabilities/tests/test_package_managers.py
+++ b/vulnerabilities/tests/test_package_managers.py
@@ -454,37 +454,24 @@ def test_fetch(self):
 
 
 class TestGoproxyVersionAPI(TestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls.version_api = GoproxyVersionAPI()
-        with open(os.path.join(TEST_DATA, "goproxy_api", "ferretdb_versions")) as f:
-            cls.vlist = f.read()
-        with open(os.path.join(TEST_DATA, "goproxy_api", "version_info")) as f:
-            cls.vinfo = json.load(f)
-        cls.expected_versions = {
-            Version(value="v0.0.1"),
-            Version(value="v0.0.2"),
-            Version(value="v0.0.3"),
-            Version(value="v0.0.4"),
-            Version(value="v0.0.5"),
-        }
-
     def test_trim_url_path(self):
         url1 = "https://pkg.go.dev/github.com/containous/traefik/v2"
         url2 = "github.com/FerretDB/FerretDB/cmd/ferretdb"
-        url3 = self.version_api.trim_url_path(url2)
-        assert "github.com/containous/traefik" == self.version_api.trim_url_path(url1)
+        url3 = GoproxyVersionAPI.trim_url_path(url2)
+        assert "github.com/containous/traefik" == GoproxyVersionAPI.trim_url_path(url1)
         assert "github.com/FerretDB/FerretDB/cmd" == url3
-        assert "github.com/FerretDB/FerretDB" == self.version_api.trim_url_path(url3)
+        assert "github.com/FerretDB/FerretDB" == GoproxyVersionAPI.trim_url_path(url3)
 
     def test_escape_path(self):
         path = "github.com/FerretDB/FerretDB"
-        assert "github.com/!ferret!d!b/!ferret!d!b" == self.version_api.escape_path(path)
+        assert "github.com/!ferret!d!b/!ferret!d!b" == GoproxyVersionAPI.escape_path(path)
 
     def test_parse_version_info(self):
-        client_session = MockClientSession(self.vinfo)
+        with open(os.path.join(TEST_DATA, "goproxy_api", "version_info")) as f:
+            vinfo = json.load(f)
+        client_session = MockClientSession(vinfo)
         assert asyncio.run(
-            self.version_api.parse_version_info(
+            GoproxyVersionAPI.parse_version_info(
                 "v0.0.5", "github.com/!ferret!d!b/!ferret!d!b", client_session
             )
         ) == Version(
@@ -493,10 +480,19 @@ def test_parse_version_info(self):
         )
 
     def test_fetch(self):
-        assert self.version_api.get("github.com/FerretDB/FerretDB") == VersionResponse()
-        client_session = MockClientSession(self.vlist)
-        asyncio.run(self.version_api.fetch("github.com/FerretDB/FerretDB", client_session))
-        assert self.version_api.cache["github.com/FerretDB/FerretDB"] == self.expected_versions
+        version_api = GoproxyVersionAPI()
+        assert version_api.get("github.com/FerretDB/FerretDB") == VersionResponse()
+        with open(os.path.join(TEST_DATA, "goproxy_api", "ferretdb_versions")) as f:
+            vlist = f.read()
+        client_session = MockClientSession(vlist)
+        asyncio.run(version_api.fetch("github.com/FerretDB/FerretDB", client_session))
+        assert version_api.cache["github.com/FerretDB/FerretDB"] == {
+            Version(value="v0.0.1"),
+            Version(value="v0.0.2"),
+            Version(value="v0.0.3"),
+            Version(value="v0.0.4"),
+            Version(value="v0.0.5"),
+        }
 
 
 class TestNugetVersionAPI(TestCase):

From f75226bb18d18b660afc1a76723aad01b2d5dd4d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Wed, 9 Feb 2022 14:49:47 +0800
Subject: [PATCH 11/14] make better docstring
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index ca9e88163..a48611304 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -510,13 +510,21 @@ class GoproxyVersionAPI(VersionAPI):
     @staticmethod
     def trim_url_path(url_path: str) -> Optional[str]:
         """
-        github advisories for golang is using package names(e.g.
-        https://github.com/advisories/GHSA-jp4j-47f9-2vc3),
-        yet goproxy works with module names(see
-        https://golang.org/ref/mod#goproxy-protocol).
-        this method removes the last part of a package path, and returns the
-        remaining as the module name. For example: trim_url_path(
-        "https://github.com/xx/a/b") returns "https://github.com/xx/a"
+        Return a trimmed Go `url_path` removing trailing
+        package references and keeping only the module
+        references.
+
+        Github advisories for Go are using package names
+        such as "https://github.com/nats-io/nats-server/v2/server"
+        (e.g., https://github.com/advisories/GHSA-jp4j-47f9-2vc3 ),
+        yet goproxy works with module names instead such as
+        "https://github.com/nats-io/nats-server" (see for details
+        https://golang.org/ref/mod#goproxy-protocol ).
+        This functions trims the trailing part(s) of a package URL
+        and returns the remaining the module name.
+        For example:
+        >>> module = "https://github.com/xx/a"
+        >>> assert trim_url_path("https://github.com/xx/a/b") == module
         """
         # some advisories contains this prefix in package name, e.g. https://github.com/advisories/GHSA-7h6j-2268-fhcm
         if url_path.startswith("https://pkg.go.dev/"):
@@ -530,7 +538,12 @@ def trim_url_path(url_path: str) -> Optional[str]:
     @staticmethod
     def escape_path(path: str) -> str:
         """
-        escape uppercase in module/version name.
+        Return an case-encoded module path or version name.
+
+        This is done by replacing every uppercase letter with an exclamation
+        mark followed by the corresponding lower-case letter, in order to
+        avoid ambiguity when serving from case-insensitive file systems.
+        Refer to https://golang.org/ref/mod#goproxy-protocol.
         """
         escaped_path = ""
         for c in path:

From aa5f8f7d10e6ced242169fb3c50414e699c81fc0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Wed, 9 Feb 2022 15:50:20 +0800
Subject: [PATCH 12/14] dedent block
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py | 35 +++++++++++++++--------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index a48611304..8781f0878 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -559,23 +559,24 @@ async def parse_version_info(
         version_info: str, escaped_pkg: str, session: ClientSession
     ) -> Optional[Version]:
         v = version_info.split()
-        if len(v) > 0:
-            value = v[0]
-            if len(v) > 1:
-                release_date = parse_datetime(v[1])
-            else:
-                escaped_ver = GoproxyVersionAPI.escape_path(value)
-                try:
-                    response = await session.request(
-                        method="GET",
-                        url=f"https://proxy.golang.org/{escaped_pkg}/@v/{escaped_ver}.info",
-                    )
-                    resp_json = await response.json()
-                    release_date = parse_datetime(resp_json.get("Time", ""))
-                except:
-                    release_date = None
-            return Version(value=value, release_date=release_date)
-        return None
+        if not v:
+            return None
+        value = v[0]
+        if len(v) > 1:
+            # get release date from the second part. see https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/proxy.go#latest()
+            release_date = parse_datetime(v[1])
+        else:
+            escaped_ver = GoproxyVersionAPI.escape_path(value)
+            try:
+                response = await session.request(
+                    method="GET",
+                    url=f"https://proxy.golang.org/{escaped_pkg}/@v/{escaped_ver}.info",
+                )
+                resp_json = await response.json()
+                release_date = parse_datetime(resp_json.get("Time", ""))
+            except:
+                release_date = None
+        return Version(value=value, release_date=release_date)
 
     async def fetch(self, pkg: str, session: ClientSession):
         # escape uppercase in module path

From b3a1432c6ae44a9559c84ec0abb18b5cb4406971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Thu, 10 Feb 2022 10:52:48 +0800
Subject: [PATCH 13/14] log network error message
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 8781f0878..43aaf884e 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -19,6 +19,7 @@
 #  for any legal advice.
 #  VulnerableCode is a free software code scanning tool from nexB Inc. and others.
 #  Visit https://github.com/nexB/vulnerablecode/ for support and download.
+import traceback
 import os
 import asyncio
 import dataclasses
@@ -575,6 +576,10 @@ async def parse_version_info(
                 resp_json = await response.json()
                 release_date = parse_datetime(resp_json.get("Time", ""))
             except:
+                traceback.print_exc()
+                print(
+                    f"error while fetching version info for {escaped_pkg}/{escaped_ver} from goproxy"
+                )
                 release_date = None
         return Version(value=value, release_date=release_date)
 

From dcc9d6ff10534be8ffa1b2ef4925445a9079f6a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E5=8F=B8=E8=8A=B3=E6=BA=90?= <sify1221@gmail.com>
Date: Thu, 10 Feb 2022 11:31:31 +0800
Subject: [PATCH 14/14] fix doctest NameError
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: 司芳源 <sify1221@gmail.com>
---
 vulnerabilities/package_managers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py
index 43aaf884e..a5c6b5540 100644
--- a/vulnerabilities/package_managers.py
+++ b/vulnerabilities/package_managers.py
@@ -525,7 +525,7 @@ def trim_url_path(url_path: str) -> Optional[str]:
         and returns the remaining the module name.
         For example:
         >>> module = "https://github.com/xx/a"
-        >>> assert trim_url_path("https://github.com/xx/a/b") == module
+        >>> assert GoproxyVersionAPI.trim_url_path("https://github.com/xx/a/b") == module
         """
         # some advisories contains this prefix in package name, e.g. https://github.com/advisories/GHSA-7h6j-2268-fhcm
         if url_path.startswith("https://pkg.go.dev/"):