From da87e3ff716832f7ab9f710b0d07253602d53445 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Mon, 18 Dec 2023 19:31:27 +0100 Subject: [PATCH] Refactor version handling code (#550) * Refactor version handling code. * Apply suggestions from code review. Co-authored-by: Maxwell G * Import sys and add docstring. * Reformat. * Document that prefer_pre is only used when pre=True. --------- Co-authored-by: Maxwell G --- src/antsibull/build_ansible_commands.py | 179 ++++------------------- src/antsibull/new_ansible.py | 121 ++-------------- src/antsibull/versions.py | 184 ++++++++++++++++++++++++ 3 files changed, 222 insertions(+), 262 deletions(-) diff --git a/src/antsibull/build_ansible_commands.py b/src/antsibull/build_ansible_commands.py index a545faf0..6b21f668 100644 --- a/src/antsibull/build_ansible_commands.py +++ b/src/antsibull/build_ansible_commands.py @@ -19,20 +19,15 @@ import aiohttp import asyncio_pool # type: ignore[import] from antsibull_core import app_context -from antsibull_core.ansible_core import ( - AnsibleCorePyPiClient, - get_ansible_core, - get_ansible_core_package_name, -) +from antsibull_core.ansible_core import get_ansible_core, get_ansible_core_package_name from antsibull_core.collections import install_together from antsibull_core.dependency_files import BuildFile, DependencyFileData, DepsFile -from antsibull_core.galaxy import CollectionDownloader, GalaxyClient +from antsibull_core.galaxy import CollectionDownloader from antsibull_core.logging import log from antsibull_core.subprocess_util import async_log_run, log_run from antsibull_core.yaml import store_yaml_file, store_yaml_stream from jinja2 import Template from packaging.version import Version as PypiVer -from semantic_version import SimpleSpec as SemVerSpec from semantic_version import Version as SemVer from antsibull.constants import MINIMUM_ANSIBLE_VERSIONS @@ -44,7 +39,13 @@ from .dep_closure import check_collection_dependencies from .tagging import get_collections_tags from .utils.get_pkg_data import get_antsibull_data -from .versions import feature_freeze_version, load_constraints_if_exists +from .versions import ( + feature_freeze_version, + find_latest_compatible, + get_latest_ansible_core_version, + get_version_info, + load_constraints_if_exists, +) if TYPE_CHECKING: from _typeshed import StrPath @@ -64,141 +65,6 @@ # -async def get_latest_ansible_core_version( - ansible_core_version: PypiVer, client: AnsibleCorePyPiClient, pre: bool = False -) -> PypiVer | None: - """ - Retrieve the latest ansible-core bugfix release's version for the given ansible-core version. - - :arg ansible_core_version: The ansible-core version. - :arg client: A AnsibleCorePyPiClient instance. - """ - all_versions = await client.get_versions() - next_version = PypiVer( - f"{ansible_core_version.major}.{ansible_core_version.minor + 1}a" - ) - newer_versions = [ - version - for version in all_versions - if ansible_core_version <= version < next_version - and (pre or not version.is_prerelease) - ] - return max(newer_versions) if newer_versions else None - - -async def get_latest_collection_version( - client: GalaxyClient, - collection: str, - version_spec: str, - pre: bool = False, - constraint: SemVerSpec | None = None, -) -> SemVer: - """ - Get the latest version of a collection that matches a specification. - - :arg collection: Namespace.collection identifying a collection. - :arg version_spec: String specifying the allowable versions. - :kwarg pre: If True, allow prereleases (versions which have the form X.Y.Z.SOMETHING). - This is **not** for excluding 0.Y.Z versions. non-pre-releases are still - preferred over pre-releases (for instance, with version_spec='>2.0.0-a1,<3.0.0' - and pre=True, if the available versions are 2.0.0-a1 and 2.0.0-a2, then 2.0.0-a2 - will be returned. If the available versions are 2.0.0 and 2.1.0-b2, 2.0.0 will be - returned since non-pre-releases are preferred. The default is False - :kwarg constraint: If provided, only consider versions that match this specification. - :returns: :obj:`semantic_version.Version` of the latest collection version that satisfied - the specification. - - .. seealso:: For the format of the version_spec, see the documentation - of :obj:`semantic_version.SimpleSpec` - """ - versions = await client.get_versions(collection) - sem_versions = [SemVer(v) for v in versions] - sem_versions.sort(reverse=True) - - spec = SemVerSpec(version_spec) - prereleases = [] - for version in (v for v in sem_versions if v in spec): - # Ignore all versions that do not match the constraints - if constraint is not None and version not in constraint: - continue - # If this is a pre-release, first check if there's a non-pre-release that - # will satisfy the version_spec. - if version.prerelease: - prereleases.append(version) - continue - return version - - # We did not find a stable version that satisies the version_spec. If we - # allow prereleases, return the latest of those here. - if pre and prereleases: - return prereleases[0] - - # No matching versions were found - constraint_clause = "" if constraint is None else f" (with constraint {constraint})" - raise ValueError( - f"{version_spec}{constraint_clause} did not match with any version of {collection}." - ) - - -async def get_collection_and_core_versions( - deps: Mapping[str, str], - ansible_core_version: PypiVer | None, - galaxy_url: str, - ansible_core_allow_prerelease: bool = False, - constraints: dict[str, SemVerSpec] | None = None, -) -> tuple[dict[str, SemVer], PypiVer | None]: - """ - Retrieve the latest version of each collection. - - :arg deps: Mapping of collection name to a version specification. - :arg ansible_core_version: Optional ansible-core version. Will search for the latest bugfix - release. - :arg galaxy_url: The url for the galaxy server to use. - :arg ansible_core_allow_prerelease: Whether to allow prereleases for ansible-core - :returns: Tuple consisting of a dict mapping collection name to latest version, and of the - ansible-core version if it was provided. - """ - constraints = constraints or {} - - requestors = {} - async with aiohttp.ClientSession() as aio_session: - lib_ctx = app_context.lib_ctx.get() - async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool: - client = GalaxyClient(aio_session, galaxy_server=galaxy_url) - for collection_name, version_spec in deps.items(): - requestors[collection_name] = await pool.spawn( - get_latest_collection_version( - client, - collection_name, - version_spec, - pre=True, - constraint=constraints.get(collection_name), - ) - ) - if ansible_core_version: - requestors["_ansible_core"] = await pool.spawn( - get_latest_ansible_core_version( - ansible_core_version, - AnsibleCorePyPiClient(aio_session), - pre=ansible_core_allow_prerelease, - ) - ) - - responses = await asyncio.gather(*requestors.values()) - - # Note: Python dicts have a stable sort order and since we haven't modified the dict since we - # used requestors.values() to generate responses, requestors and responses therefore have - # a matching order. - included_versions: dict[str, SemVer] = {} - for collection_name, version in zip(requestors, responses): - if collection_name == "_ansible_core": - ansible_core_version = version - else: - included_versions[collection_name] = version - - return included_versions, ansible_core_version - - async def download_collections( versions: Mapping[str, SemVer], galaxy_url: str, @@ -509,18 +375,31 @@ def prepare_command() -> int: for collection_name, spec in old_deps.items(): deps[collection_name] = feature_freeze_version(spec, collection_name) - included_versions, new_ansible_core_version = asyncio.run( - get_collection_and_core_versions( - deps, - ansible_core_version_obj, - str(app_ctx.galaxy_url), - ansible_core_allow_prerelease=_is_alpha(app_ctx.extra["ansible_version"]), - constraints=constraints, + ansible_core_release_infos, collections_to_versions = asyncio.run( + get_version_info( + list(deps), + pypi_server_url=app_ctx.pypi_url, + galaxy_url=str(app_ctx.galaxy_url), ) ) + + new_ansible_core_version = get_latest_ansible_core_version( + list(ansible_core_release_infos), + ansible_core_version_obj, + pre=_is_alpha(app_ctx.extra["ansible_version"]), + ) if new_ansible_core_version: ansible_core_version_obj = new_ansible_core_version + included_versions = find_latest_compatible( + ansible_core_version_obj, + collections_to_versions, + version_specs=deps, + pre=True, + prefer_pre=False, + constraints=constraints, + ) + if not str(app_ctx.extra["ansible_version"]).startswith(build_ansible_version): print( f"{build_filename} is for version {build_ansible_version} but we need" diff --git a/src/antsibull/new_ansible.py b/src/antsibull/new_ansible.py index b91b79af..0f437a5d 100644 --- a/src/antsibull/new_ansible.py +++ b/src/antsibull/new_ansible.py @@ -9,121 +9,17 @@ import asyncio import os -import typing as t -from collections.abc import Mapping, Sequence -import aiohttp -import asyncio_pool # type: ignore[import] -import semantic_version as semver from antsibull_core import app_context -from antsibull_core.ansible_core import AnsibleCorePyPiClient from antsibull_core.dependency_files import BuildFile, parse_pieces_file -from antsibull_core.galaxy import GalaxyClient from packaging.version import Version as PypiVer from .changelog import ChangelogData -from .versions import load_constraints_if_exists - - -def display_exception(loop, context): # pylint:disable=unused-argument - print(context.get("exception")) - - -async def get_version_info( - collections: Sequence[str], pypi_server_url: str -) -> tuple[dict[str, t.Any], dict[str, list[str]]]: - """ - Return the versions of all the collections and ansible-core - """ - loop = asyncio.get_running_loop() - loop.set_exception_handler(display_exception) - - requestors = {} - async with aiohttp.ClientSession() as aio_session: - lib_ctx = app_context.lib_ctx.get() - async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool: - pypi_client = AnsibleCorePyPiClient( - aio_session, pypi_server_url=pypi_server_url - ) - requestors["_ansible_core"] = await pool.spawn( - pypi_client.get_release_info() - ) - galaxy_client = GalaxyClient(aio_session) - - for collection in collections: - requestors[collection] = await pool.spawn( - galaxy_client.get_versions(collection) - ) - - collection_versions = {} - responses = await asyncio.gather(*requestors.values()) - - ansible_core_release_infos: dict[str, t.Any] | None = None - for idx, collection_name in enumerate(requestors): - if collection_name == "_ansible_core": - ansible_core_release_infos = responses[idx] - else: - collection_versions[collection_name] = responses[idx] - - if ansible_core_release_infos is None: - raise RuntimeError("Internal error") - - return ansible_core_release_infos, collection_versions - - -def version_is_compatible( - # pylint:disable-next=unused-argument - ansible_core_version: PypiVer, - # pylint:disable-next=unused-argument - collection: str, - version: semver.Version, - allow_prereleases: bool = False, - constraint: semver.SimpleSpec | None = None, -) -> bool: - # Metadata for this is not currently implemented. So everything is rated as compatible - # as long as it is no prerelease - if version.prerelease and not allow_prereleases: - return False - if constraint is not None and version not in constraint: - return False - return True - - -def find_latest_compatible( - ansible_core_version: PypiVer, - raw_dependency_versions: Mapping[str, Sequence[str]], - allow_prereleases: bool = False, - constraints: Mapping[str, semver.SimpleSpec] | None = None, -) -> dict[str, semver.Version]: - # Note: ansible-core compatibility is not currently implemented. It will be a piece of - # collection metadata that is present in the collection but may not be present in galaxy. - # We'll have to figure that out once the pieces are finalized - - constraints = constraints or {} - - # Order versions - reduced_versions = {} - for dep, versions in raw_dependency_versions.items(): - # Order the versions - versions = [semver.Version(v) for v in versions] - versions.sort(reverse=True) - - # Step through the versions to select the latest one which is compatible - for version in versions: - if version_is_compatible( - ansible_core_version, - dep, - version, - allow_prereleases=allow_prereleases, - constraint=constraints.get(dep), - ): - reduced_versions[dep] = version - break - - if dep not in reduced_versions: - raise ValueError(f"Cannot find matching version for {dep}") - - return reduced_versions +from .versions import ( + find_latest_compatible, + get_version_info, + load_constraints_if_exists, +) def new_ansible_command() -> int: @@ -131,7 +27,7 @@ def new_ansible_command() -> int: collections = parse_pieces_file( os.path.join(app_ctx.extra["data_dir"], app_ctx.extra["pieces_file"]) ) - ansible_core_release_infos, dependencies = asyncio.run( + ansible_core_release_infos, collections_to_versions = asyncio.run( get_version_info(collections, str(app_ctx.pypi_url)) ) ansible_core_versions = [ @@ -148,8 +44,9 @@ def new_ansible_command() -> int: ansible_core_version, python_requires = ansible_core_versions[0] dependencies = find_latest_compatible( ansible_core_version, - dependencies, - allow_prereleases=app_ctx.extra["allow_prereleases"], + collections_to_versions, + pre=app_ctx.extra["allow_prereleases"], + prefer_pre=True, constraints=constraints, ) diff --git a/src/antsibull/versions.py b/src/antsibull/versions.py index 77c795d1..884f4fc5 100644 --- a/src/antsibull/versions.py +++ b/src/antsibull/versions.py @@ -7,9 +7,19 @@ from __future__ import annotations +import asyncio import os +import sys +import typing as t +from collections.abc import Mapping, Sequence +import aiohttp +import asyncio_pool # type: ignore[import] +from antsibull_core import app_context +from antsibull_core.ansible_core import AnsibleCorePyPiClient from antsibull_core.dependency_files import parse_pieces_file +from antsibull_core.galaxy import GalaxyClient +from packaging.version import Version as PypiVer from semantic_version import SimpleSpec as SemVerSpec from semantic_version import Version as SemVer @@ -142,3 +152,177 @@ def load_constraints_if_exists(filename: str) -> dict[str, SemVerSpec]: ) from exc result[collection] = constraint return result + + +def _display_exception(loop, context): # pylint:disable=unused-argument + print(context.get("exception"), file=sys.stderr) + + +async def get_version_info( + collections: Sequence[str], + pypi_server_url: str | None = None, + galaxy_url: str | None = None, +) -> tuple[dict[str, t.Any], dict[str, list[str]]]: + """ + Return the versions of all the collections and ansible-core + """ + loop = asyncio.get_running_loop() + loop.set_exception_handler(_display_exception) + + requestors = {} + lib_ctx = app_context.lib_ctx.get() + async with ( + aiohttp.ClientSession() as aio_session, + asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool, + ): + pypi_client = AnsibleCorePyPiClient( + aio_session, pypi_server_url=pypi_server_url + ) + requestors["_ansible_core"] = await pool.spawn(pypi_client.get_release_info()) + + galaxy_client = GalaxyClient(aio_session, galaxy_server=galaxy_url) + for collection in collections: + requestors[collection] = await pool.spawn( + galaxy_client.get_versions(collection) + ) + + collections_to_versions = {} + responses = await asyncio.gather(*requestors.values()) + + ansible_core_release_infos: dict[str, t.Any] | None = None + for idx, collection_name in enumerate(requestors): + if collection_name == "_ansible_core": + ansible_core_release_infos = responses[idx] + else: + collections_to_versions[collection_name] = responses[idx] + + if ansible_core_release_infos is None: + raise RuntimeError("Internal error") + + return ansible_core_release_infos, collections_to_versions + + +def get_latest_ansible_core_version( + ansible_core_versions: Sequence[str], + ansible_core_version: PypiVer, + pre: bool = False, +) -> PypiVer | None: + """ + Retrieve the latest ansible-core bugfix release's version for the given ansible-core version. + + :arg ansible_core_versions: A list of ansible-core versions. + :arg ansible_core_version: A ansible-core version. + """ + versions = [PypiVer(v) for v in ansible_core_versions] + next_version = PypiVer( + f"{ansible_core_version.major}.{ansible_core_version.minor + 1}a" + ) + newer_versions = [ + version + for version in versions + if ansible_core_version <= version < next_version + and (pre or not version.is_prerelease) + ] + return max(newer_versions) if newer_versions else None + + +def get_latest_collection_version( + versions: Sequence[str], + collection: str, + version_spec: str | None = None, + pre: bool = False, + prefer_pre: bool = False, + constraint: SemVerSpec | None = None, +) -> SemVer: + """ + Get the latest version of a collection that matches a specification. + + :arg versions: Sequence of collection versions + :arg collection: Namespace.collection identifying a collection. + :arg version_spec: Optional string specifying the allowable versions. + :kwarg pre: If ``True``, allow prereleases (versions which have the form X.Y.Z.SOMETHING). + This is **not** for excluding 0.Y.Z versions. Non-pre-releases are still + preferred over pre-releases, except if ``prefer_pre=True`` (for instance, with + ``version_spec='>2.0.0-a1,<3.0.0'`` and ``pre=True``, if the available versions + are 2.0.0-a1 and 2.0.0-a2, then 2.0.0-a2 will be returned. If the available + versions are 2.0.0 and 2.1.0-b2, 2.0.0 will be returned since non-pre-releases + are preferred.) The default is ``False``. + :kwarg prefer_pre: If ``True``, prefer newer pre-releases over stable releases. Is only + used when ``pre=True``. + :kwarg constraint: If provided, only consider versions that match this specification. + :returns: :obj:`semantic_version.Version` of the latest collection version that satisfied + the specification. + + .. seealso:: For the format of the version_spec, see the documentation + of :obj:`semantic_version.SimpleSpec` + """ + sem_versions = [SemVer(v) for v in versions] + sem_versions.sort(reverse=True) + + if version_spec: + spec = SemVerSpec(version_spec) + sem_versions = [v for v in sem_versions if v in spec] + + if constraint: + # Ignore all versions that do not match the constraints + sem_versions = [v for v in sem_versions if v in constraint] + + prereleases = [] + for version in sem_versions: + # If this is a pre-release, first check if there's a non-pre-release that + # will satisfy the version_spec. + if version.prerelease: + # If we prefer prereleases, take this one. + if pre and prefer_pre: + return version + prereleases.append(version) + continue + return version + + # We did not find a stable version that satisies the version_spec. If we + # allow prereleases, return the latest of those here. + if pre and prereleases: + return prereleases[0] + + # No matching versions were found + constraint_clause = "" if constraint is None else f" (with constraint {constraint})" + raise ValueError( + f"{version_spec}{constraint_clause} did not match with any version of {collection}." + ) + + +def find_latest_compatible( + ansible_core_version: PypiVer, # pylint: disable=unused-argument + collections_to_versions: Mapping[str, Sequence[str]], + pre: bool = False, + prefer_pre: bool = False, + version_specs: Mapping[str, str] | None = None, + constraints: Mapping[str, SemVerSpec] | None = None, +) -> dict[str, SemVer]: + """ + Finds the latest compatible version of every collection from ``collections_to_versions`` + that matches the given ``vresion_specs`` and ``constraints``. + + ``pre`` and ``prefer_pre`` control whether pre-releases are acceptable (``pre=True``), + and in case both matching pre-releases and releases are found, which ones to prefer + (``prefer_pre=True`` prefers pre-releases over regular releases). + """ + # Note: ansible-core compatibility is not currently implemented. It will be a piece of + # collection metadata that is present in the collection but may not be present in Galaxy. + # We'll have to figure that out once the pieces are finalized + + version_specs = version_specs or {} + constraints = constraints or {} + + reduced_versions = {} + for dep, versions in collections_to_versions.items(): + reduced_versions[dep] = get_latest_collection_version( + versions, + dep, + version_spec=version_specs.get(dep), + pre=pre, + prefer_pre=prefer_pre, + constraint=constraints.get(dep), + ) + + return reduced_versions