From fbe53654b31a2eea29355612908930aa08e9e3ae Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Mon, 13 May 2024 20:55:32 +0200 Subject: [PATCH] `prepare` subcommand: allow to preserve existing `.deps` files (#599) * Add --preserve-deps option. * Add antsibull_preserve_deps option for the release role. * Add more docstrings. --- changelogs/fragments/599-prepare-preserve.yml | 7 + roles/build-release/defaults/main.yaml | 3 + roles/build-release/meta/argument_specs.yml | 8 ++ roles/build-release/tasks/build.yaml | 1 + src/antsibull/build_ansible_commands.py | 126 +++++++++++++----- src/antsibull/cli/antsibull_build.py | 10 ++ 6 files changed, 122 insertions(+), 33 deletions(-) create mode 100644 changelogs/fragments/599-prepare-preserve.yml diff --git a/changelogs/fragments/599-prepare-preserve.yml b/changelogs/fragments/599-prepare-preserve.yml new file mode 100644 index 00000000..005c619f --- /dev/null +++ b/changelogs/fragments/599-prepare-preserve.yml @@ -0,0 +1,7 @@ +minor_changes: + - "Add option ``--preserve-deps`` to the ``prepare`` subcommand that allows to preserve the + dependencies if a ``.deps`` file for that version already exists. The versions from that + ``.deps`` file are validated against the build requirements and constraints, and the + remainder of the release preparation process remains unchanged. The release role allows + to pass this flag when ``antsibull_preserve_deps=true`` + (https://github.com/ansible-community/antsibull/pull/599)." diff --git a/roles/build-release/defaults/main.yaml b/roles/build-release/defaults/main.yaml index 5874ab82..be147077 100644 --- a/roles/build-release/defaults/main.yaml +++ b/roles/build-release/defaults/main.yaml @@ -44,6 +44,9 @@ antsibull_ansible_venv: "{{ antsibull_sdist_dir }}/venv" # Whether or not to start from scratch with a new venv if one exists antsibull_venv_cleanup: true +# Whether to preserve existing .deps files during the prepare step +antsibull_preserve_deps: false + ##### # These variables relate to verifying that collections properly tag their diff --git a/roles/build-release/meta/argument_specs.yml b/roles/build-release/meta/argument_specs.yml index 8afe03b5..af3a5166 100644 --- a/roles/build-release/meta/argument_specs.yml +++ b/roles/build-release/meta/argument_specs.yml @@ -136,3 +136,11 @@ argument_specs: - Defaults to 0 (all available CPUs) type: int default: 0 + + antsibull_preserve_deps: + description: + - If set to V(true), will preserve existing C(.deps) files during the preparation + process and validate their contents against the build requirements and constraints. + type: bool + default: false + version_added: 0.62.0 diff --git a/roles/build-release/tasks/build.yaml b/roles/build-release/tasks/build.yaml index 367b3041..d06b1caf 100644 --- a/roles/build-release/tasks/build.yaml +++ b/roles/build-release/tasks/build.yaml @@ -55,6 +55,7 @@ --data-dir {{ antsibull_data_dir }} {{ _feature_freeze | default('') }} {{ '--tags-file' if antsibull_tags_validate else '' }} + {{ '--preserve-deps' if antsibull_preserve_deps else '' }} # Minimal failure tolerance to galaxy collection download errors retries: 3 delay: 5 diff --git a/src/antsibull/build_ansible_commands.py b/src/antsibull/build_ansible_commands.py index 04c90b6a..6e59b34a 100644 --- a/src/antsibull/build_ansible_commands.py +++ b/src/antsibull/build_ansible_commands.py @@ -28,6 +28,7 @@ from antsibull_core.yaml import store_yaml_file, store_yaml_stream from jinja2 import Template from packaging.version import Version as PypiVer +from semantic_version import SimpleSpec as SemVerSpec from semantic_version import Version as SemVer from antsibull.constants import MINIMUM_ANSIBLE_VERSIONS @@ -356,34 +357,21 @@ def _extract_python_requires( ) -def prepare_command() -> int: - app_ctx = app_context.app_ctx.get() +def prepare_deps( + ansible_version: PypiVer, + ansible_core_version_obj: PypiVer, + build_deps: dict[str, str], + constraints: dict[str, SemVerSpec], +) -> DependencyFileData: + """ + Collect ansible-core and collection versions for a new release. + """ lib_ctx = app_context.lib_ctx.get() - build_filename = os.path.join( - app_ctx.extra["data_dir"], app_ctx.extra["build_file"] - ) - build_file = BuildFile(build_filename) - build_ansible_version, ansible_core_version, deps = build_file.parse() - ansible_core_version_obj = PypiVer(ansible_core_version) - python_requires = _extract_python_requires(ansible_core_version_obj, deps) - - constraints_filename = os.path.join( - app_ctx.extra["data_dir"], app_ctx.extra["constraints_file"] - ) - constraints = load_constraints_if_exists(constraints_filename) - - # If we're building a feature frozen release (betas and rcs) then we need to - # change the upper version limit to not include new features. - if app_ctx.extra["feature_frozen"]: - old_deps, deps = deps, {} - for collection_name, spec in old_deps.items(): - deps[collection_name] = feature_freeze_version(spec, collection_name) - galaxy_context = asyncio.run(create_galaxy_context()) ansible_core_release_infos, collections_to_versions = asyncio.run( get_version_info( - list(deps), + list(build_deps), pypi_server_url=str(lib_ctx.pypi_url), galaxy_context=galaxy_context, ) @@ -392,7 +380,7 @@ def prepare_command() -> int: new_ansible_core_version = get_latest_ansible_core_version( list(ansible_core_release_infos), ansible_core_version_obj, - pre=_is_alpha(app_ctx.extra["ansible_version"]), + pre=_is_alpha(ansible_version), ) if new_ansible_core_version: ansible_core_version_obj = new_ansible_core_version @@ -400,12 +388,57 @@ def prepare_command() -> int: included_versions = find_latest_compatible( ansible_core_version_obj, collections_to_versions, - version_specs=deps, + version_specs=build_deps, pre=True, prefer_pre=False, constraints=constraints, ) + return DependencyFileData( + str(ansible_version), + str(ansible_core_version_obj), + {collection: str(version) for collection, version in included_versions.items()}, + ) + + +def validate_deps_data( + deps: dict[str, str], + build_deps: dict[str, str], + constraints: dict[str, SemVerSpec], +) -> None: + """ + Validate dependencies against constraints and build deps. + + Raise ``ValueError`` in case of inconsistencies. + """ + for dependency, version in sorted(deps.items()): + version_obj = SemVer(version) + if dependency in build_deps: + spec = SemVerSpec(build_deps[dependency]) + if version_obj not in spec: + raise ValueError( + f"The build dependencies for {dependency} require" + f" {version} to be in {spec}" + ) + if dependency in constraints: + if version_obj not in constraints[dependency]: + raise ValueError( + f"The constraints for {dependency} require" + f" {version} to be in {constraints[dependency]}" + ) + + +def prepare_command() -> int: + app_ctx = app_context.app_ctx.get() + + build_filename = os.path.join( + app_ctx.extra["data_dir"], app_ctx.extra["build_file"] + ) + build_file = BuildFile(build_filename) + build_ansible_version, ansible_core_version, build_deps = build_file.parse() + ansible_core_version_obj = PypiVer(ansible_core_version) + python_requires = _extract_python_requires(ansible_core_version_obj, build_deps) + if not str(app_ctx.extra["ansible_version"]).startswith(build_ansible_version): print( f"{build_filename} is for version {build_ansible_version} but we need" @@ -414,11 +447,42 @@ def prepare_command() -> int: ) return 1 - dependency_data = DependencyFileData( - str(app_ctx.extra["ansible_version"]), - str(ansible_core_version_obj), - {collection: str(version) for collection, version in included_versions.items()}, + constraints_filename = os.path.join( + app_ctx.extra["data_dir"], app_ctx.extra["constraints_file"] + ) + constraints = load_constraints_if_exists(constraints_filename) + + # If we're building a feature frozen release (betas and rcs) then we need to + # change the upper version limit to not include new features. + if app_ctx.extra["feature_frozen"]: + build_deps = { + collection_name: feature_freeze_version(spec, collection_name) + for collection_name, spec in build_deps.items() + } + + deps_filename = os.path.join( + app_ctx.extra["dest_data_dir"], app_ctx.extra["deps_file"] ) + deps_file = DepsFile(deps_filename) + + if app_ctx.extra["preserve_deps"] and os.path.exists(deps_filename): + dependency_data = deps_file.parse() + else: + dependency_data = prepare_deps( + app_ctx.extra["ansible_version"], + ansible_core_version_obj, + build_deps, + constraints, + ) + + try: + validate_deps_data(dependency_data.deps, build_deps, constraints) + except ValueError as exc: + print( + "Error while validating existing dependencies against" + f" build version ranges and constraints: {exc}" + ) + return 1 # Get Ansible changelog, add new release ansible_changelog = ChangelogData.ansible( @@ -436,10 +500,6 @@ def prepare_command() -> int: ansible_changelog.changes.save() # Write dependency file - deps_filename = os.path.join( - app_ctx.extra["dest_data_dir"], app_ctx.extra["deps_file"] - ) - deps_file = DepsFile(deps_filename) deps_file.write( dependency_data.ansible_version, dependency_data.ansible_core_version, diff --git a/src/antsibull/cli/antsibull_build.py b/src/antsibull/cli/antsibull_build.py index ef773474..07cebe47 100644 --- a/src/antsibull/cli/antsibull_build.py +++ b/src/antsibull/cli/antsibull_build.py @@ -392,6 +392,14 @@ def parse_args(program_name: str, args: list[str]) -> argparse.Namespace: " is $BASENAME_OF_BUILD_FILE-X.Y.Z.yaml", ) + preserve_deps_parser = argparse.ArgumentParser(add_help=False) + preserve_deps_parser.add_argument( + "--preserve-deps", + action="store_true", + help="If this is given and the deps file already exists, use it" + " instead of creating a new one.", + ) + # Delay import to avoid potential import loops from antsibull import __version__ as _ver # pylint: disable=import-outside-toplevel @@ -453,6 +461,7 @@ def parse_args(program_name: str, args: list[str]) -> argparse.Namespace: build_step_parser, feature_freeze_parser, galaxy_file_parser, + preserve_deps_parser, ], description="Collect dependencies for an Ansible release", ) @@ -474,6 +483,7 @@ def parse_args(program_name: str, args: list[str]) -> argparse.Namespace: build_step_parser, feature_freeze_parser, galaxy_file_parser, + preserve_deps_parser, ], description="Build a single-file Ansible" " [deprecated]", )