Add support for metadata retrieval from GHCR.io

.github/workflows/tests.yml

@@ -29,14 +29,15 @@ jobs:
           use-mamba: true
 
       - name: configure conda and install code
-        shell: bash -l {0}
+        shell: bash -el {0}
         run: |
-          mamba install --yes --file=requirements.txt
-          mamba install --yes --file=requirements-dev.txt
+          mamba install --yes \
+            --file=requirements.txt \
+            --file=requirements-dev.txt
           python -m pip install -v --no-deps --no-build-isolation -e .
 
       - name: test versions
-        shell: bash -l {0}
+        shell: bash -el {0}
         run: |
           pip uninstall conda-forge-metadata --yes
           [[ $(python setup.py --version) != "0.0.0" ]] || exit 1
@@ -60,6 +61,6 @@ jobs:
           python -m pip install -v --no-deps --no-build-isolation -e .
 
       - name: test
-        shell: bash -l {0}
+        shell: bash -el {0}
         run: |
           pytest -vvs tests

.pre-commit-config.yaml

@@ -46,7 +46,7 @@ repos:
   rev: v2.32.0
   hooks:
   - id: pyupgrade
-    args: ["--py310-plus", "--keep-runtime-typing"]
+    args: ["--py38-plus", "--keep-runtime-typing"]
 
 # Organize imports.
 - repo: https://github.com/pre-commit/mirrors-isort

conda_forge_metadata/artifact_info/info_json.py

@@ -1,7 +1,7 @@
 from conda_forge_metadata.libcfgraph import get_libcfgraph_artifact_data
 
 
-def get_artifact_info_as_json(channel, subdir, artifact):
+def get_artifact_info_as_json(channel, subdir, artifact, backend="libcfgraph"):
     """Get a blob of artifact data from the conda info directory.
 
     Parameters
@@ -35,4 +35,11 @@ def get_artifact_info_as_json(channel, subdir, artifact):
             "files": a list of files in the recipe from info/files with
                 elements ending in .pyc or .txt filtered out.
     """
-    return get_libcfgraph_artifact_data(channel, subdir, artifact)
+    if backend == "libcfgraph":
+        return get_libcfgraph_artifact_data(channel, subdir, artifact)
+    elif backend == "oci":
+        from conda_forge_metadata.oci import get_oci_artifact_data
+
+        return get_oci_artifact_data(channel, subdir, artifact)
+    else:
+        raise ValueError(f"Unknown backend {backend!r}")

conda_forge_metadata/autotick_bot/pypi_to_conda.py

@@ -1,7 +1,7 @@
 from functools import lru_cache
 
 import requests
-import yaml
+from ruamel import yaml
 
 
 @lru_cache(maxsize=1)
@@ -11,7 +11,7 @@ def get_pypi_name_mapping():
         "master/mappings/pypi/name_mapping.yaml"
     )
     req.raise_for_status()
-    return yaml.safe_load(req.text)
+    return yaml.YAML(typ="safe").load(req.text)
 
 
 @lru_cache(maxsize=1)

conda_forge_metadata/oci.py

@@ -0,0 +1,112 @@
+import json
+from functools import lru_cache
+from logging import getLogger
+from typing import Union
+
+from conda_oci_mirror.repo import PackageRepo
+from ruamel import yaml
+
+logger = getLogger(__name__)
+
+
+def _extract_read(infotar, *names) -> Union[str, None]:
+    names_in_tar = infotar.getnames()
+    for name in names:
+        if name in names_in_tar:
+            return infotar.extractfile(name).read().decode()
+
+
+@lru_cache(maxsize=1024)
+def get_oci_artifact_data(
+    channel: str,
+    subdir: str,
+    artifact: str,
+    registry: str = "ghcr.io/channel-mirrors",
+) -> Union[dict, None]:
+    """Get a blob of artifact data from the conda info directory.
+
+    Note this function might need token authentication to access the registry.
+    Export the following environment variables:
+    - ORAS_USER: the username to use for authentication
+    - ORAS_PASS: the password/token to use for authentication
+
+    Parameters
+    ----------
+    channel : str
+        The channel (e.g., "conda-forge").
+    subdir : str
+        The subdir for the artifact (e.g., "noarch", "linux-64", etc.).
+    artifact : str
+        The full artifact name with extension (e.g.,
+        "21cmfast-3.0.2-py36h13dd421_0.tar.bz2").
+    registry : str
+        The registry to use for the OCI repository.
+
+    Returns
+    -------
+    info_blob : dict
+        A dictionary of data. Possible keys are
+
+            "metadata_version": the metadata version format
+            "name": the package name
+            "version": the package version
+            "index": the info/index.json file contents
+            "about": the info/about.json file contents
+            "rendered_recipe": the fully rendered recipe at
+                either info/recipe/meta.yaml or info/meta.yaml
+                as a dict
+            "raw_recipe": the template recipe as a string from
+                info/recipe/meta.yaml.template - could be
+                the rendered recipe as a string if no template was found
+            "conda_build_config": the conda_build_config.yaml used for building
+                the recipe at info/recipe/conda_build_config.yaml
+            "files": a list of files in the recipe from info/files with
+                elements ending in .pyc or .txt filtered out.
+
+        If the artifact is not indexed, it returns None.
+    """
+    if artifact.endswith(".tar.bz2"):
+        artifact = artifact[: -len(".tar.bz2")]
+    elif artifact.endswith(".conda"):
+        artifact = artifact[: -len(".conda")]
+    else:
+        raise ValueError(f"Artifact '{artifact}' is not a conda package")
+
+    repo = PackageRepo(channel, subdir, None, registry=registry)
+
+    parts = artifact.rsplit("-", 2)
+    oci_name = f"{parts[0]}:{parts[1]}-{parts[2]}"
+    try:
+        infotar = repo.get_info(oci_name)
+    except ValueError as exc:
+        logger.debug("Failed to get info for %s", oci_name, exc_info=exc)
+        return None
+
+    YAML = yaml.YAML(typ="safe")
+
+    index = json.loads(_extract_read(infotar, "index.json"))
+    return {
+        # https://github.com/regro/libcflib/blob/062858e90af2795d2eb098034728cace574a51b8/libcflib/harvester.py#L14
+        "metadata_version": 1,
+        "name": index.get("name", ""),
+        "version": index.get("version", ""),
+        "index": index,
+        "about": json.loads(_extract_read(infotar, "about.json")),
+        "rendered_recipe": YAML.load(
+            _extract_read(infotar, "recipe/meta.yaml", "meta.yaml")
+        ),
+        "raw_recipe": _extract_read(
+            infotar,
+            "recipe/meta.yaml.template",
+            "recipe/meta.yaml",
+            "meta.yaml",
+        ),
+        "conda_build_config": YAML.load(
+            _extract_read(infotar, "recipe/conda_build_config.yaml")
+        ),
+        "files": [
+            f
+            for f in _extract_read(infotar, "files").splitlines()
+            if not f.lower().endswith((".pyc", ".txt"))
+        ],
+    }

pyproject.toml

@@ -15,11 +15,16 @@ description = "programatic access to conda-forge's metadata"
 dynamic = ["version"]
 dependencies = [
     "requests",
-    "pyyaml"
+    "ruamel.yaml"
 ]
 license = {file = "LICENSE"}
 readme = "README.md"
 
+[project.optional-dependencies]
+oci = [
+  "conda-oci-mirror@git+https://github.com/channel-mirrors/[email protected]#egg=conda-oci-mirror"
+]
+
 [project.urls]
 home = "https://github.com/regro/conda-forge-metadata"
 

requirements.txt

@@ -1,2 +1,3 @@
-pyyaml
+conda-oci-mirror
 requests
+ruamel.yaml

tests/test_info_json.py

@@ -0,0 +1,30 @@
+import pytest
+
+from conda_forge_metadata.artifact_info import info_json
+
+
+@pytest.mark.parametrize("backend", ["libcfgraph", "oci"])
+def test_info_json(backend):
+    info = info_json.get_artifact_info_as_json(
+        "conda-forge",
+        "osx-64",
+        "21cmfast-3.0.2-py36h13dd421_0.tar.bz2",
+        backend=backend,
+    )
+    assert info["metadata_version"] == 1
+    assert info["name"] == "21cmfast"
+    assert info["version"] == "3.0.2"
+    assert info["index"]["name"] == "21cmfast"
+    assert info["index"]["version"] == "3.0.2"
+    assert info["index"]["build"] == "py36h13dd421_0"
+    assert info["index"]["subdir"] == "osx-64"
+    assert "pyyaml" in info["index"]["depends"]
+    assert info["about"]["conda_version"] == "4.8.4"
+    assert info["rendered_recipe"]["package"]["name"] == "21cmfast"
+    assert (
+        info["rendered_recipe"]["source"]["sha256"]
+        == "6e88960d134e98e4719343d853c63fc3c691438b57b2863f7834f07fae9eab4f"
+    )
+    assert info["raw_recipe"].startswith('{% set name = "21cmFAST" %}')
+    assert info["conda_build_config"]["CI"] == "azure"
+    assert "bin/21cmfast" in info["files"]