diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 57ecb6f..bead3f7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,14 +29,15 @@ jobs: use-mamba: true - name: configure conda and install code - shell: bash -l {0} + shell: bash -el {0} run: | - mamba install --yes --file=requirements.txt - mamba install --yes --file=requirements-dev.txt + mamba install --yes \ + --file=requirements.txt \ + --file=requirements-dev.txt python -m pip install -v --no-deps --no-build-isolation -e . - name: test versions - shell: bash -l {0} + shell: bash -el {0} run: | pip uninstall conda-forge-metadata --yes [[ $(python setup.py --version) != "0.0.0" ]] || exit 1 @@ -60,6 +61,6 @@ jobs: python -m pip install -v --no-deps --no-build-isolation -e . - name: test - shell: bash -l {0} + shell: bash -el {0} run: | pytest -vvs tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0931baa..7c245b8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: rev: v2.32.0 hooks: - id: pyupgrade - args: ["--py310-plus", "--keep-runtime-typing"] + args: ["--py38-plus", "--keep-runtime-typing"] # Organize imports. - repo: https://github.com/pre-commit/mirrors-isort diff --git a/conda_forge_metadata/artifact_info/info_json.py b/conda_forge_metadata/artifact_info/info_json.py index ade2cbf..aef2572 100644 --- a/conda_forge_metadata/artifact_info/info_json.py +++ b/conda_forge_metadata/artifact_info/info_json.py @@ -1,7 +1,7 @@ from conda_forge_metadata.libcfgraph import get_libcfgraph_artifact_data -def get_artifact_info_as_json(channel, subdir, artifact): +def get_artifact_info_as_json(channel, subdir, artifact, backend="libcfgraph"): """Get a blob of artifact data from the conda info directory. Parameters @@ -35,4 +35,11 @@ def get_artifact_info_as_json(channel, subdir, artifact): "files": a list of files in the recipe from info/files with elements ending in .pyc or .txt filtered out. """ - return get_libcfgraph_artifact_data(channel, subdir, artifact) + if backend == "libcfgraph": + return get_libcfgraph_artifact_data(channel, subdir, artifact) + elif backend == "oci": + from conda_forge_metadata.oci import get_oci_artifact_data + + return get_oci_artifact_data(channel, subdir, artifact) + else: + raise ValueError(f"Unknown backend {backend!r}") diff --git a/conda_forge_metadata/autotick_bot/pypi_to_conda.py b/conda_forge_metadata/autotick_bot/pypi_to_conda.py index 3da50d0..224374a 100644 --- a/conda_forge_metadata/autotick_bot/pypi_to_conda.py +++ b/conda_forge_metadata/autotick_bot/pypi_to_conda.py @@ -1,7 +1,7 @@ from functools import lru_cache import requests -import yaml +from ruamel import yaml @lru_cache(maxsize=1) @@ -11,7 +11,7 @@ def get_pypi_name_mapping(): "master/mappings/pypi/name_mapping.yaml" ) req.raise_for_status() - return yaml.safe_load(req.text) + return yaml.YAML(typ="safe").load(req.text) @lru_cache(maxsize=1) diff --git a/conda_forge_metadata/oci.py b/conda_forge_metadata/oci.py new file mode 100644 index 0000000..8a43c9f --- /dev/null +++ b/conda_forge_metadata/oci.py @@ -0,0 +1,112 @@ +import json +from functools import lru_cache +from logging import getLogger +from typing import Union + +from conda_oci_mirror.repo import PackageRepo +from ruamel import yaml + +logger = getLogger(__name__) + + +def _extract_read(infotar, *names) -> Union[str, None]: + names_in_tar = infotar.getnames() + for name in names: + if name in names_in_tar: + return infotar.extractfile(name).read().decode() + + +@lru_cache(maxsize=1024) +def get_oci_artifact_data( + channel: str, + subdir: str, + artifact: str, + registry: str = "ghcr.io/channel-mirrors", +) -> Union[dict, None]: + """Get a blob of artifact data from the conda info directory. + + Note this function might need token authentication to access the registry. + Export the following environment variables: + - ORAS_USER: the username to use for authentication + - ORAS_PASS: the password/token to use for authentication + + Parameters + ---------- + channel : str + The channel (e.g., "conda-forge"). + subdir : str + The subdir for the artifact (e.g., "noarch", "linux-64", etc.). + artifact : str + The full artifact name with extension (e.g., + "21cmfast-3.0.2-py36h13dd421_0.tar.bz2"). + registry : str + The registry to use for the OCI repository. + + Returns + ------- + info_blob : dict + A dictionary of data. Possible keys are + + "metadata_version": the metadata version format + "name": the package name + "version": the package version + "index": the info/index.json file contents + "about": the info/about.json file contents + "rendered_recipe": the fully rendered recipe at + either info/recipe/meta.yaml or info/meta.yaml + as a dict + "raw_recipe": the template recipe as a string from + info/recipe/meta.yaml.template - could be + the rendered recipe as a string if no template was found + "conda_build_config": the conda_build_config.yaml used for building + the recipe at info/recipe/conda_build_config.yaml + "files": a list of files in the recipe from info/files with + elements ending in .pyc or .txt filtered out. + + If the artifact is not indexed, it returns None. + """ + if artifact.endswith(".tar.bz2"): + artifact = artifact[: -len(".tar.bz2")] + elif artifact.endswith(".conda"): + artifact = artifact[: -len(".conda")] + else: + raise ValueError(f"Artifact '{artifact}' is not a conda package") + + repo = PackageRepo(channel, subdir, None, registry=registry) + + parts = artifact.rsplit("-", 2) + oci_name = f"{parts[0]}:{parts[1]}-{parts[2]}" + try: + infotar = repo.get_info(oci_name) + except ValueError as exc: + logger.debug("Failed to get info for %s", oci_name, exc_info=exc) + return None + + YAML = yaml.YAML(typ="safe") + + index = json.loads(_extract_read(infotar, "index.json")) + return { + # https://github.com/regro/libcflib/blob/062858e90af2795d2eb098034728cace574a51b8/libcflib/harvester.py#L14 + "metadata_version": 1, + "name": index.get("name", ""), + "version": index.get("version", ""), + "index": index, + "about": json.loads(_extract_read(infotar, "about.json")), + "rendered_recipe": YAML.load( + _extract_read(infotar, "recipe/meta.yaml", "meta.yaml") + ), + "raw_recipe": _extract_read( + infotar, + "recipe/meta.yaml.template", + "recipe/meta.yaml", + "meta.yaml", + ), + "conda_build_config": YAML.load( + _extract_read(infotar, "recipe/conda_build_config.yaml") + ), + "files": [ + f + for f in _extract_read(infotar, "files").splitlines() + if not f.lower().endswith((".pyc", ".txt")) + ], + } diff --git a/pyproject.toml b/pyproject.toml index c0a7fde..eb0fbd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,11 +15,16 @@ description = "programatic access to conda-forge's metadata" dynamic = ["version"] dependencies = [ "requests", - "pyyaml" + "ruamel.yaml" ] license = {file = "LICENSE"} readme = "README.md" +[project.optional-dependencies] +oci = [ + "conda-oci-mirror@git+https://github.com/channel-mirrors/conda-oci-mirror.git@v0.1.0#egg=conda-oci-mirror" +] + [project.urls] home = "https://github.com/regro/conda-forge-metadata" diff --git a/requirements.txt b/requirements.txt index 1c6d8b4..3d7b6df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -pyyaml +conda-oci-mirror requests +ruamel.yaml diff --git a/tests/test_info_json.py b/tests/test_info_json.py new file mode 100644 index 0000000..eead188 --- /dev/null +++ b/tests/test_info_json.py @@ -0,0 +1,30 @@ +import pytest + +from conda_forge_metadata.artifact_info import info_json + + +@pytest.mark.parametrize("backend", ["libcfgraph", "oci"]) +def test_info_json(backend): + info = info_json.get_artifact_info_as_json( + "conda-forge", + "osx-64", + "21cmfast-3.0.2-py36h13dd421_0.tar.bz2", + backend=backend, + ) + assert info["metadata_version"] == 1 + assert info["name"] == "21cmfast" + assert info["version"] == "3.0.2" + assert info["index"]["name"] == "21cmfast" + assert info["index"]["version"] == "3.0.2" + assert info["index"]["build"] == "py36h13dd421_0" + assert info["index"]["subdir"] == "osx-64" + assert "pyyaml" in info["index"]["depends"] + assert info["about"]["conda_version"] == "4.8.4" + assert info["rendered_recipe"]["package"]["name"] == "21cmfast" + assert ( + info["rendered_recipe"]["source"]["sha256"] + == "6e88960d134e98e4719343d853c63fc3c691438b57b2863f7834f07fae9eab4f" + ) + assert info["raw_recipe"].startswith('{% set name = "21cmFAST" %}') + assert info["conda_build_config"]["CI"] == "azure" + assert "bin/21cmfast" in info["files"]