Merge pull request #156 from notatallshaw/fix-up-py2index-script

Metadata files, skip 404, and PEP 723 support for py2ndex

tests/functional/python/py2index.py

@@ -1,3 +1,12 @@
+# /// script
+# requires-python = ">=3.7"
+# dependencies = [
+#   "requests",
+#   "packaging",
+#   "html5lib"
+# ]
+# ///
+
 """Freeze metadata from Python index server to test locally.
 
 Inspired by index_from_rubygems.rb from CocoaPods/Resolver-Integration-Specs.
@@ -20,10 +29,7 @@
 import re
 import sys
 import urllib.parse
-import zipfile
 from typing import (
-    IO,
-    BinaryIO,
     Dict,
     FrozenSet,
     Iterable,
@@ -34,7 +40,6 @@
     Set,
     Tuple,
     Union,
-    cast,
 )
 
 import html5lib
@@ -198,15 +203,6 @@ def _parse_wheel_name(rest: str) -> Tuple[str, str, str]:
     return name, version, f"{x}-{y}-{z}"
 
 
-def _open_metadata(zf: zipfile.ZipFile, prefix: str) -> IO[bytes]:
-    for fn in zf.namelist():
-        if not fn.endswith(".dist-info/METADATA"):
-            continue
-        if packaging.utils.canonicalize_name(fn).startswith(prefix):
-            return zf.open(fn)
-    raise ValueError("Can't find metadata")
-
-
 class PackageEntry(NamedTuple):
     version: str
     dependencies: List[str]
@@ -221,10 +217,12 @@ class Finder:
     matcher: WheelMatcher
     session: requests.Session
 
-    def collect_best_dist_urls(self, name: str) -> Dict[str, str]:
+    def collect_best_metadta_urls(self, name: str) -> Dict[str, str]:
         all_dists: DistListMapping = collections.defaultdict(list)
         for index_url in self.index_urls:
             res = requests.get(f"{index_url}/{name}")
+            if res.status_code == 404:
+                logger.critical("Project %s does not exist", name)
             res.raise_for_status()
             doc = html5lib.parse(res.content, namespaceHTMLElements=False)
             for el in doc.findall(".//a"):
@@ -247,18 +245,20 @@ def collect_best_dist_urls(self, name: str) -> Dict[str, str]:
                     continue
                 if rank is None:
                     continue
+
+                url = urllib.parse.urljoin(
+                    url, urllib.parse.urlparse(url).path + ".metadata"
+                )
                 all_dists[version].append((rank, url))
         urls = {version: min(dists)[1] for version, dists in all_dists.items()}
         logger.info("%d URLs found for %s", len(urls), name)
         return urls
 
     def iter_package_entries(self, name: str) -> Iterator[PackageEntry]:
-        for version, url in self.collect_best_dist_urls(name).items():
-            http_file = cast(IO[bytes], HttpFile(url, self.session))
-            with zipfile.ZipFile(http_file) as zf:
-                with _open_metadata(zf, name) as f:
-                    parser = email.parser.BytesParser()
-                    data = parser.parse(cast(BinaryIO, f), headersonly=True)
+        for version, url in self.collect_best_metadta_urls(name).items():
+            http_file = HttpFile(url, self.session)
+            parser = email.parser.BytesParser()
+            data = parser.parsebytes(http_file.read(), headersonly=True)
             dependencies: List[str] = data.get_all("Requires-Dist", [])
             yield PackageEntry(version, dependencies)