prepare subcommand: allow to preserve existing .deps files (#599)

* Add --preserve-deps option.

* Add antsibull_preserve_deps option for the release role.

* Add more docstrings.

changelogs/fragments/599-prepare-preserve.yml

@@ -0,0 +1,7 @@
+minor_changes:
+  - "Add option ``--preserve-deps`` to the ``prepare`` subcommand that allows to preserve the
+     dependencies if a ``.deps`` file for that version already exists. The versions from that
+     ``.deps`` file are validated against the build requirements and constraints, and the
+     remainder of the release preparation process remains unchanged. The release role allows
+     to pass this flag when ``antsibull_preserve_deps=true``
+     (https://github.com/ansible-community/antsibull/pull/599)."

roles/build-release/defaults/main.yaml

@@ -44,6 +44,9 @@ antsibull_ansible_venv: "{{ antsibull_sdist_dir }}/venv"
 # Whether or not to start from scratch with a new venv if one exists
 antsibull_venv_cleanup: true
 
+# Whether to preserve existing .deps files during the prepare step
+antsibull_preserve_deps: false
+
 
 #####
 # These variables relate to verifying that collections properly tag their

roles/build-release/meta/argument_specs.yml

@@ -136,3 +136,11 @@ argument_specs:
           - Defaults to 0 (all available CPUs)
         type: int
         default: 0
+
+      antsibull_preserve_deps:
+        description:
+          - If set to V(true), will preserve existing C(.deps) files during the preparation
+            process and validate their contents against the build requirements and constraints.
+        type: bool
+        default: false
+        version_added: 0.62.0

roles/build-release/tasks/build.yaml

@@ -55,6 +55,7 @@
       --data-dir {{ antsibull_data_dir }}
       {{ _feature_freeze | default('') }}
       {{ '--tags-file' if antsibull_tags_validate else '' }}
+      {{ '--preserve-deps' if antsibull_preserve_deps else '' }}
   # Minimal failure tolerance to galaxy collection download errors
   retries: 3
   delay: 5

src/antsibull/build_ansible_commands.py

@@ -28,6 +28,7 @@
 from antsibull_core.yaml import store_yaml_file, store_yaml_stream
 from jinja2 import Template
 from packaging.version import Version as PypiVer
+from semantic_version import SimpleSpec as SemVerSpec
 from semantic_version import Version as SemVer
 
 from antsibull.constants import MINIMUM_ANSIBLE_VERSIONS
@@ -356,34 +357,21 @@ def _extract_python_requires(
     )
 
 
-def prepare_command() -> int:
-    app_ctx = app_context.app_ctx.get()
+def prepare_deps(
+    ansible_version: PypiVer,
+    ansible_core_version_obj: PypiVer,
+    build_deps: dict[str, str],
+    constraints: dict[str, SemVerSpec],
+) -> DependencyFileData:
+    """
+    Collect ansible-core and collection versions for a new release.
+    """
     lib_ctx = app_context.lib_ctx.get()
 
-    build_filename = os.path.join(
-        app_ctx.extra["data_dir"], app_ctx.extra["build_file"]
-    )
-    build_file = BuildFile(build_filename)
-    build_ansible_version, ansible_core_version, deps = build_file.parse()
-    ansible_core_version_obj = PypiVer(ansible_core_version)
-    python_requires = _extract_python_requires(ansible_core_version_obj, deps)
-
-    constraints_filename = os.path.join(
-        app_ctx.extra["data_dir"], app_ctx.extra["constraints_file"]
-    )
-    constraints = load_constraints_if_exists(constraints_filename)
-
-    # If we're building a feature frozen release (betas and rcs) then we need to
-    # change the upper version limit to not include new features.
-    if app_ctx.extra["feature_frozen"]:
-        old_deps, deps = deps, {}
-        for collection_name, spec in old_deps.items():
-            deps[collection_name] = feature_freeze_version(spec, collection_name)
-
     galaxy_context = asyncio.run(create_galaxy_context())
     ansible_core_release_infos, collections_to_versions = asyncio.run(
         get_version_info(
-            list(deps),
+            list(build_deps),
             pypi_server_url=str(lib_ctx.pypi_url),
             galaxy_context=galaxy_context,
         )
@@ -392,20 +380,65 @@ def prepare_command() -> int:
     new_ansible_core_version = get_latest_ansible_core_version(
         list(ansible_core_release_infos),
         ansible_core_version_obj,
-        pre=_is_alpha(app_ctx.extra["ansible_version"]),
+        pre=_is_alpha(ansible_version),
     )
     if new_ansible_core_version:
         ansible_core_version_obj = new_ansible_core_version
 
     included_versions = find_latest_compatible(
         ansible_core_version_obj,
         collections_to_versions,
-        version_specs=deps,
+        version_specs=build_deps,
         pre=True,
         prefer_pre=False,
         constraints=constraints,
     )
 
+    return DependencyFileData(
+        str(ansible_version),
+        str(ansible_core_version_obj),
+        {collection: str(version) for collection, version in included_versions.items()},
+    )
+
+
+def validate_deps_data(
+    deps: dict[str, str],
+    build_deps: dict[str, str],
+    constraints: dict[str, SemVerSpec],
+) -> None:
+    """
+    Validate dependencies against constraints and build deps.
+
+    Raise ``ValueError`` in case of inconsistencies.
+    """
+    for dependency, version in sorted(deps.items()):
+        version_obj = SemVer(version)
+        if dependency in build_deps:
+            spec = SemVerSpec(build_deps[dependency])
+            if version_obj not in spec:
+                raise ValueError(
+                    f"The build dependencies for {dependency} require"
+                    f" {version} to be in {spec}"
+                )
+        if dependency in constraints:
+            if version_obj not in constraints[dependency]:
+                raise ValueError(
+                    f"The constraints for {dependency} require"
+                    f" {version} to be in {constraints[dependency]}"
+                )
+
+
+def prepare_command() -> int:
+    app_ctx = app_context.app_ctx.get()
+
+    build_filename = os.path.join(
+        app_ctx.extra["data_dir"], app_ctx.extra["build_file"]
+    )
+    build_file = BuildFile(build_filename)
+    build_ansible_version, ansible_core_version, build_deps = build_file.parse()
+    ansible_core_version_obj = PypiVer(ansible_core_version)
+    python_requires = _extract_python_requires(ansible_core_version_obj, build_deps)
+
     if not str(app_ctx.extra["ansible_version"]).startswith(build_ansible_version):
         print(
             f"{build_filename} is for version {build_ansible_version} but we need"
@@ -414,11 +447,42 @@ def prepare_command() -> int:
         )
         return 1
 
-    dependency_data = DependencyFileData(
-        str(app_ctx.extra["ansible_version"]),
-        str(ansible_core_version_obj),
-        {collection: str(version) for collection, version in included_versions.items()},
+    constraints_filename = os.path.join(
+        app_ctx.extra["data_dir"], app_ctx.extra["constraints_file"]
+    )
+    constraints = load_constraints_if_exists(constraints_filename)
+
+    # If we're building a feature frozen release (betas and rcs) then we need to
+    # change the upper version limit to not include new features.
+    if app_ctx.extra["feature_frozen"]:
+        build_deps = {
+            collection_name: feature_freeze_version(spec, collection_name)
+            for collection_name, spec in build_deps.items()
+        }
+
+    deps_filename = os.path.join(
+        app_ctx.extra["dest_data_dir"], app_ctx.extra["deps_file"]
     )
+    deps_file = DepsFile(deps_filename)
+
+    if app_ctx.extra["preserve_deps"] and os.path.exists(deps_filename):
+        dependency_data = deps_file.parse()
+    else:
+        dependency_data = prepare_deps(
+            app_ctx.extra["ansible_version"],
+            ansible_core_version_obj,
+            build_deps,
+            constraints,
+        )
+
+    try:
+        validate_deps_data(dependency_data.deps, build_deps, constraints)
+    except ValueError as exc:
+        print(
+            "Error while validating existing dependencies against"
+            f" build version ranges and constraints: {exc}"
+        )
+        return 1
 
     # Get Ansible changelog, add new release
     ansible_changelog = ChangelogData.ansible(
@@ -436,10 +500,6 @@ def prepare_command() -> int:
     ansible_changelog.changes.save()
 
     # Write dependency file
-    deps_filename = os.path.join(
-        app_ctx.extra["dest_data_dir"], app_ctx.extra["deps_file"]
-    )
-    deps_file = DepsFile(deps_filename)
     deps_file.write(
         dependency_data.ansible_version,
         dependency_data.ansible_core_version,

src/antsibull/cli/antsibull_build.py

@@ -392,6 +392,14 @@ def parse_args(program_name: str, args: list[str]) -> argparse.Namespace:
         " is $BASENAME_OF_BUILD_FILE-X.Y.Z.yaml",
     )
 
+    preserve_deps_parser = argparse.ArgumentParser(add_help=False)
+    preserve_deps_parser.add_argument(
+        "--preserve-deps",
+        action="store_true",
+        help="If this is given and the deps file already exists, use it"
+        " instead of creating a new one.",
+    )
+
     # Delay import to avoid potential import loops
     from antsibull import __version__ as _ver  # pylint: disable=import-outside-toplevel
 
@@ -453,6 +461,7 @@ def parse_args(program_name: str, args: list[str]) -> argparse.Namespace:
             build_step_parser,
             feature_freeze_parser,
             galaxy_file_parser,
+            preserve_deps_parser,
         ],
         description="Collect dependencies for an Ansible release",
     )
@@ -474,6 +483,7 @@ def parse_args(program_name: str, args: list[str]) -> argparse.Namespace:
             build_step_parser,
             feature_freeze_parser,
             galaxy_file_parser,
+            preserve_deps_parser,
         ],
         description="Build a single-file Ansible" " [deprecated]",
     )