Skip to content

Commit

Permalink
Refactor version handling code (#550)
Browse files Browse the repository at this point in the history 
* Refactor version handling code.

* Apply suggestions from code review.

Co-authored-by: Maxwell G <[email protected]>

* Import sys and add docstring.

* Reformat.

* Document that prefer_pre is only used when pre=True.

---------

Co-authored-by: Maxwell G <[email protected]>
  • Loading branch information
@felixfontein @gotmax23
felixfontein and gotmax23 authored Dec 18, 2023
1 parent 3d48af6 commit da87e3f
Show file tree
Hide file tree
Showing 3 changed files with 222 additions and 262 deletions.
179 changes: 29 additions & 150 deletions src/antsibull/build_ansible_commands.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,15 @@
import aiohttp
import asyncio_pool # type: ignore[import]
from antsibull_core import app_context
from antsibull_core.ansible_core import (
AnsibleCorePyPiClient,
get_ansible_core,
get_ansible_core_package_name,
)
from antsibull_core.ansible_core import get_ansible_core, get_ansible_core_package_name
from antsibull_core.collections import install_together
from antsibull_core.dependency_files import BuildFile, DependencyFileData, DepsFile
from antsibull_core.galaxy import CollectionDownloader, GalaxyClient
from antsibull_core.galaxy import CollectionDownloader
from antsibull_core.logging import log
from antsibull_core.subprocess_util import async_log_run, log_run
from antsibull_core.yaml import store_yaml_file, store_yaml_stream
from jinja2 import Template
from packaging.version import Version as PypiVer
from semantic_version import SimpleSpec as SemVerSpec
from semantic_version import Version as SemVer

from antsibull.constants import MINIMUM_ANSIBLE_VERSIONS
Expand All @@ -44,7 +39,13 @@
from .dep_closure import check_collection_dependencies
from .tagging import get_collections_tags
from .utils.get_pkg_data import get_antsibull_data
from .versions import feature_freeze_version, load_constraints_if_exists
from .versions import (
feature_freeze_version,
find_latest_compatible,
get_latest_ansible_core_version,
get_version_info,
load_constraints_if_exists,
)

if TYPE_CHECKING:
from _typeshed import StrPath
Expand All @@ -64,141 +65,6 @@
#


async def get_latest_ansible_core_version(
ansible_core_version: PypiVer, client: AnsibleCorePyPiClient, pre: bool = False
) -> PypiVer | None:
"""
Retrieve the latest ansible-core bugfix release's version for the given ansible-core version.
:arg ansible_core_version: The ansible-core version.
:arg client: A AnsibleCorePyPiClient instance.
"""
all_versions = await client.get_versions()
next_version = PypiVer(
f"{ansible_core_version.major}.{ansible_core_version.minor + 1}a"
)
newer_versions = [
version
for version in all_versions
if ansible_core_version <= version < next_version
and (pre or not version.is_prerelease)
]
return max(newer_versions) if newer_versions else None


async def get_latest_collection_version(
client: GalaxyClient,
collection: str,
version_spec: str,
pre: bool = False,
constraint: SemVerSpec | None = None,
) -> SemVer:
"""
Get the latest version of a collection that matches a specification.
:arg collection: Namespace.collection identifying a collection.
:arg version_spec: String specifying the allowable versions.
:kwarg pre: If True, allow prereleases (versions which have the form X.Y.Z.SOMETHING).
This is **not** for excluding 0.Y.Z versions. non-pre-releases are still
preferred over pre-releases (for instance, with version_spec='>2.0.0-a1,<3.0.0'
and pre=True, if the available versions are 2.0.0-a1 and 2.0.0-a2, then 2.0.0-a2
will be returned. If the available versions are 2.0.0 and 2.1.0-b2, 2.0.0 will be
returned since non-pre-releases are preferred. The default is False
:kwarg constraint: If provided, only consider versions that match this specification.
:returns: :obj:`semantic_version.Version` of the latest collection version that satisfied
the specification.
.. seealso:: For the format of the version_spec, see the documentation
of :obj:`semantic_version.SimpleSpec`
"""
versions = await client.get_versions(collection)
sem_versions = [SemVer(v) for v in versions]
sem_versions.sort(reverse=True)

spec = SemVerSpec(version_spec)
prereleases = []
for version in (v for v in sem_versions if v in spec):
# Ignore all versions that do not match the constraints
if constraint is not None and version not in constraint:
continue
# If this is a pre-release, first check if there's a non-pre-release that
# will satisfy the version_spec.
if version.prerelease:
prereleases.append(version)
continue
return version

# We did not find a stable version that satisies the version_spec. If we
# allow prereleases, return the latest of those here.
if pre and prereleases:
return prereleases[0]

# No matching versions were found
constraint_clause = "" if constraint is None else f" (with constraint {constraint})"
raise ValueError(
f"{version_spec}{constraint_clause} did not match with any version of {collection}."
)


async def get_collection_and_core_versions(
deps: Mapping[str, str],
ansible_core_version: PypiVer | None,
galaxy_url: str,
ansible_core_allow_prerelease: bool = False,
constraints: dict[str, SemVerSpec] | None = None,
) -> tuple[dict[str, SemVer], PypiVer | None]:
"""
Retrieve the latest version of each collection.
:arg deps: Mapping of collection name to a version specification.
:arg ansible_core_version: Optional ansible-core version. Will search for the latest bugfix
release.
:arg galaxy_url: The url for the galaxy server to use.
:arg ansible_core_allow_prerelease: Whether to allow prereleases for ansible-core
:returns: Tuple consisting of a dict mapping collection name to latest version, and of the
ansible-core version if it was provided.
"""
constraints = constraints or {}

requestors = {}
async with aiohttp.ClientSession() as aio_session:
lib_ctx = app_context.lib_ctx.get()
async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool:
client = GalaxyClient(aio_session, galaxy_server=galaxy_url)
for collection_name, version_spec in deps.items():
requestors[collection_name] = await pool.spawn(
get_latest_collection_version(
client,
collection_name,
version_spec,
pre=True,
constraint=constraints.get(collection_name),
)
)
if ansible_core_version:
requestors["_ansible_core"] = await pool.spawn(
get_latest_ansible_core_version(
ansible_core_version,
AnsibleCorePyPiClient(aio_session),
pre=ansible_core_allow_prerelease,
)
)

responses = await asyncio.gather(*requestors.values())

# Note: Python dicts have a stable sort order and since we haven't modified the dict since we
# used requestors.values() to generate responses, requestors and responses therefore have
# a matching order.
included_versions: dict[str, SemVer] = {}
for collection_name, version in zip(requestors, responses):
if collection_name == "_ansible_core":
ansible_core_version = version
else:
included_versions[collection_name] = version

return included_versions, ansible_core_version


async def download_collections(
versions: Mapping[str, SemVer],
galaxy_url: str,
Expand Down Expand Up @@ -509,18 +375,31 @@ def prepare_command() -> int:
for collection_name, spec in old_deps.items():
deps[collection_name] = feature_freeze_version(spec, collection_name)

included_versions, new_ansible_core_version = asyncio.run(
get_collection_and_core_versions(
deps,
ansible_core_version_obj,
str(app_ctx.galaxy_url),
ansible_core_allow_prerelease=_is_alpha(app_ctx.extra["ansible_version"]),
constraints=constraints,
ansible_core_release_infos, collections_to_versions = asyncio.run(
get_version_info(
list(deps),
pypi_server_url=app_ctx.pypi_url,
galaxy_url=str(app_ctx.galaxy_url),
)
)

new_ansible_core_version = get_latest_ansible_core_version(
list(ansible_core_release_infos),
ansible_core_version_obj,
pre=_is_alpha(app_ctx.extra["ansible_version"]),
)
if new_ansible_core_version:
ansible_core_version_obj = new_ansible_core_version

included_versions = find_latest_compatible(
ansible_core_version_obj,
collections_to_versions,
version_specs=deps,
pre=True,
prefer_pre=False,
constraints=constraints,
)

if not str(app_ctx.extra["ansible_version"]).startswith(build_ansible_version):
print(
f"{build_filename} is for version {build_ansible_version} but we need"
Expand Down
121 changes: 9 additions & 112 deletions src/antsibull/new_ansible.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,129 +9,25 @@

import asyncio
import os
import typing as t
from collections.abc import Mapping, Sequence

import aiohttp
import asyncio_pool # type: ignore[import]
import semantic_version as semver
from antsibull_core import app_context
from antsibull_core.ansible_core import AnsibleCorePyPiClient
from antsibull_core.dependency_files import BuildFile, parse_pieces_file
from antsibull_core.galaxy import GalaxyClient
from packaging.version import Version as PypiVer

from .changelog import ChangelogData
from .versions import load_constraints_if_exists


def display_exception(loop, context): # pylint:disable=unused-argument
print(context.get("exception"))


async def get_version_info(
collections: Sequence[str], pypi_server_url: str
) -> tuple[dict[str, t.Any], dict[str, list[str]]]:
"""
Return the versions of all the collections and ansible-core
"""
loop = asyncio.get_running_loop()
loop.set_exception_handler(display_exception)

requestors = {}
async with aiohttp.ClientSession() as aio_session:
lib_ctx = app_context.lib_ctx.get()
async with asyncio_pool.AioPool(size=lib_ctx.thread_max) as pool:
pypi_client = AnsibleCorePyPiClient(
aio_session, pypi_server_url=pypi_server_url
)
requestors["_ansible_core"] = await pool.spawn(
pypi_client.get_release_info()
)
galaxy_client = GalaxyClient(aio_session)

for collection in collections:
requestors[collection] = await pool.spawn(
galaxy_client.get_versions(collection)
)

collection_versions = {}
responses = await asyncio.gather(*requestors.values())

ansible_core_release_infos: dict[str, t.Any] | None = None
for idx, collection_name in enumerate(requestors):
if collection_name == "_ansible_core":
ansible_core_release_infos = responses[idx]
else:
collection_versions[collection_name] = responses[idx]

if ansible_core_release_infos is None:
raise RuntimeError("Internal error")

return ansible_core_release_infos, collection_versions


def version_is_compatible(
# pylint:disable-next=unused-argument
ansible_core_version: PypiVer,
# pylint:disable-next=unused-argument
collection: str,
version: semver.Version,
allow_prereleases: bool = False,
constraint: semver.SimpleSpec | None = None,
) -> bool:
# Metadata for this is not currently implemented. So everything is rated as compatible
# as long as it is no prerelease
if version.prerelease and not allow_prereleases:
return False
if constraint is not None and version not in constraint:
return False
return True


def find_latest_compatible(
ansible_core_version: PypiVer,
raw_dependency_versions: Mapping[str, Sequence[str]],
allow_prereleases: bool = False,
constraints: Mapping[str, semver.SimpleSpec] | None = None,
) -> dict[str, semver.Version]:
# Note: ansible-core compatibility is not currently implemented. It will be a piece of
# collection metadata that is present in the collection but may not be present in galaxy.
# We'll have to figure that out once the pieces are finalized

constraints = constraints or {}

# Order versions
reduced_versions = {}
for dep, versions in raw_dependency_versions.items():
# Order the versions
versions = [semver.Version(v) for v in versions]
versions.sort(reverse=True)

# Step through the versions to select the latest one which is compatible
for version in versions:
if version_is_compatible(
ansible_core_version,
dep,
version,
allow_prereleases=allow_prereleases,
constraint=constraints.get(dep),
):
reduced_versions[dep] = version
break

if dep not in reduced_versions:
raise ValueError(f"Cannot find matching version for {dep}")

return reduced_versions
from .versions import (
find_latest_compatible,
get_version_info,
load_constraints_if_exists,
)


def new_ansible_command() -> int:
app_ctx = app_context.app_ctx.get()
collections = parse_pieces_file(
os.path.join(app_ctx.extra["data_dir"], app_ctx.extra["pieces_file"])
)
ansible_core_release_infos, dependencies = asyncio.run(
ansible_core_release_infos, collections_to_versions = asyncio.run(
get_version_info(collections, str(app_ctx.pypi_url))
)
ansible_core_versions = [
Expand All @@ -148,8 +44,9 @@ def new_ansible_command() -> int:
ansible_core_version, python_requires = ansible_core_versions[0]
dependencies = find_latest_compatible(
ansible_core_version,
dependencies,
allow_prereleases=app_ctx.extra["allow_prereleases"],
collections_to_versions,
pre=app_ctx.extra["allow_prereleases"],
prefer_pre=True,
constraints=constraints,
)

Expand Down
Loading

0 comments on commit da87e3f

Please sign in to comment.