Better fix for add-back-references (#33207)

This is a better fix for add-back-references breeze command, as
the previous fix had a bug and processed more than just providers
when providers --gen-type was selected. See #33149.

This version fixes it in a much more consistent way:

* you have to specify what you want to work on as option
* you can  select few of the provider via auto-complete
* you can also select apache-airflow and helm-chart
* you can mix and match them if you need

All the scripts and references where `--gen-type` was used
previously have been updated and documentation was changed to
reflect it

(cherry picked from commit 8c28454)

BREEZE.rst

@@ -2051,23 +2051,35 @@ To add back references to the documentation generated by ``build-docs`` in Breez
 use the ``release-management add-back-references`` command. This is important to support backward compatibility
 the airflow documentation.
 
-.. code-block:: bash
-
-     breeze release-management add-back-references
+You have to specify which packages you run it on. For example you can run it for all providers:
 
 .. code-block:: bash
 
-     release-management add-back-references --airflow-site-directory DIRECTORY
+     release-management add-back-references --airflow-site-directory DIRECTORY all-providers
 
 The flag ``--airflow-site-directory`` takes the path of the cloned ``airflow-site``. The command will
 not proceed if this is an invalid path.
 
+You can also run the command for apache-airflow (core documentation):
+
+.. code-block:: bash
+
+     breeze release-management publish-docs --airflow-site-directory DIRECTORY apache-airflow
+
+Also for helm-chart package:
+
+.. code-block:: bash
+
+     breeze release-management publish-docs --airflow-site-directory DIRECTORY helm-chart
+
+
+You can also manually specify (it's auto-completable) list of packages to run the command for including individual
+providers - you can mix apache-airflow, helm-chart and provider packages this way:
+
 .. code-block:: bash
 
-     breeze release-management publish-docs --gen-type [airflow | providers | helm]
+     breeze release-management publish-docs --airflow-site-directory DIRECTORY apache.airflow apache.beam google
 
-The flag ``--gen-type`` is a string of which type of back references to generate. It can take the values of
-airflow, providers, or helm. The command will fail if this is incorrect.
 
 These are all available flags of ``release-management add-back-references`` command:
 

dev/README_RELEASE_AIRFLOW.md

@@ -817,7 +817,7 @@ Documentation for providers can be found in the ``/docs/apache-airflow`` directo
 
     ```shell script
     breeze release-management publish-docs --package-filter apache-airflow --package-filter docker-stack
-    breeze release-management add-back-references --gen-type airflow
+    breeze release-management add-back-references apache-airflow
     git add .
     git commit -m "Add documentation for Apache Airflow ${VERSION}"
     git push

dev/README_RELEASE_HELM_CHART.md

@@ -633,7 +633,7 @@ between the two repositories to be able to build the documentation.
   Regenerate `index.yaml` so it can be added to the Airflow website to allow: `helm repo add https://airflow.apache.org`.
 
     ```shell
-    breeze release-management add-back-references --gen-type helm
+    breeze release-management add-back-references helm-chart
     curl https://dist.apache.org/repos/dist/dev/airflow/helm-chart/$RC/index.yaml -o index.yaml
     cp ${AIRFLOW_SVN_RELEASE_HELM}/${VERSION}/airflow-${VERSION}.tgz .
     helm repo index --merge ./index.yaml . --url "https://downloads.apache.org/airflow/helm-chart/$VERSION"

dev/README_RELEASE_PROVIDER_PACKAGES.md

@@ -392,7 +392,7 @@ breeze release-management publish-docs \
     --package-filter 'apache-airflow-providers-*' \
     --override-versioned
 
-breeze release-management add-back-references --gen-type providers
+breeze release-management add-back-references all-providers
 ```
 
 If you see `ModuleNotFoundError: No module named 'docs'`, set:

dev/breeze/src/airflow_breeze/commands/developer_commands.py

@@ -34,7 +34,7 @@
     DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
     DOCKER_DEFAULT_PLATFORM,
     MOUNT_SELECTED,
-    get_available_documentation_provider_packages,
+    get_available_documentation_packages,
 )
 from airflow_breeze.params.build_ci_params import BuildCiParams
 from airflow_breeze.params.doc_build_params import DocBuildParams
@@ -362,7 +362,7 @@ def start_airflow(
 @click.option(
     "--package-filter",
     help="List of packages to consider.",
-    type=NotVerifiedBetterChoice(get_available_documentation_provider_packages()),
+    type=NotVerifiedBetterChoice(get_available_documentation_packages()),
     multiple=True,
 )
 @click.option(

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -43,16 +43,16 @@
     MOUNT_ALL,
     MOUNT_SELECTED,
     MULTI_PLATFORM,
-    get_available_documentation_provider_packages,
+    get_available_documentation_packages,
 )
 from airflow_breeze.params.shell_params import ShellParams
 from airflow_breeze.utils.add_back_references import (
-    GenerationType,
     start_generating_back_references,
 )
 from airflow_breeze.utils.ci_group import ci_group
 from airflow_breeze.utils.common_options import (
     argument_packages,
+    argument_packages_plus_all_providers,
     option_airflow_constraints_mode_ci,
     option_airflow_constraints_reference,
     option_airflow_extras,
@@ -783,7 +783,7 @@ def alias_image(image_from: str, image_to: str):
 @click.option(
     "--package-filter",
     help="List of packages to consider.",
-    type=NotVerifiedBetterChoice(get_available_documentation_provider_packages()),
+    type=NotVerifiedBetterChoice(get_available_documentation_packages()),
     multiple=True,
 )
 @option_verbose
@@ -815,7 +815,7 @@ def publish_docs(
 
 @release_management.command(
     name="add-back-references",
-    help="Command to add back references for documentation to make it backward compatible",
+    help="Command to add back references for documentation to make it backward compatible.",
 )
 @click.option(
     "-a",
@@ -825,23 +825,12 @@ def publish_docs(
     help="Local directory path of cloned airflow-site repo.",
     required=True,
 )
-@click.option(
-    "-g",
-    "--gen-type",
-    show_default=True,
-    help="Type of back references to generate. Forced to providers if providers specified as arguments.",
-    type=BetterChoice(
-        [e.name for e in GenerationType],
-    ),
-    default=GenerationType.airflow.name,
-)
-@argument_packages
+@argument_packages_plus_all_providers
 @option_verbose
 @option_dry_run
 def add_back_references(
     airflow_site_directory: str,
-    gen_type: str,
-    packages: list[str],
+    packages_plus_all_providers: tuple[str],
 ):
     """Adds back references for documentation generated by build-docs and publish-docs"""
     site_path = Path(airflow_site_directory)
@@ -851,22 +840,16 @@ def add_back_references(
             "Provide the path of cloned airflow-site repo\n"
         )
         sys.exit(1)
-    if len(packages) != 0 and gen_type != GenerationType.providers.name:
+    if len(packages_plus_all_providers) == 0:
         get_console().print(
-            [
-                f"[warning]Forcing gen type to "
-                f"{GenerationType.providers} as some provider_packages are selected."
-            ]
-        )
-        gen_type = GenerationType.providers.name
-    gen = GenerationType[gen_type]
-    if gen not in GenerationType:
-        get_console().print(
-            "\n[error]invalid type of doc generation required. Pass one of [airflow | providers | helm]\n"
+            "\n[error]You need to specify at least one package to generate back references for\n"
         )
         sys.exit(1)
-
-    start_generating_back_references(gen, site_path, packages)
+    packages = list(packages_plus_all_providers)
+    if "all-providers" in packages_plus_all_providers:
+        packages.remove("all-providers")
+        packages.extend(get_available_documentation_packages(only_providers=True, short_version=True))
+    start_generating_back_references(site_path, packages)
 
 
 @release_management.command(

dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py

@@ -174,7 +174,7 @@
     "breeze release-management add-back-references": [
         {
             "name": "Add Back References to Docs",
-            "options": ["--airflow-site-directory", "--gen-type"],
+            "options": ["--airflow-site-directory"],
         },
     ],
     "breeze release-management generate-issue-content-providers": [

dev/breeze/src/airflow_breeze/global_constants.py

@@ -146,18 +146,20 @@ def all_helm_test_packages() -> list[str]:
 ALL_HISTORICAL_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
 
 
-def get_available_documentation_provider_packages(short_version=False) -> list[str]:
+def get_available_documentation_packages(short_version=False, only_providers: bool = False) -> list[str]:
     provider_names: list[str] = list(json.loads(PROVIDER_DEPENDENCIES_JSON_FILE_PATH.read_text()).keys())
     doc_provider_names = [provider_name.replace(".", "-") for provider_name in provider_names]
-    available_packages = [f"apache-airflow-providers-{doc_provider}" for doc_provider in doc_provider_names]
-    available_packages.extend(["apache-airflow", "docker-stack", "helm-chart"])
-    available_packages.sort()
+    available_packages = []
+    if not only_providers:
+        available_packages.extend(["apache-airflow", "docker-stack", "helm-chart"])
+    all_providers = [f"apache-airflow-providers-{doc_provider}" for doc_provider in doc_provider_names]
+    all_providers.sort()
+    available_packages.extend(all_providers)
     if short_version:
         prefix_len = len("apache-airflow-providers-")
         available_packages = [
-            package[prefix_len:].replace("-", ".")
+            package[prefix_len:].replace("-", ".") if len(package) > prefix_len else package
             for package in available_packages
-            if len(package) > prefix_len
         ]
     return available_packages
 

dev/breeze/src/airflow_breeze/utils/add_back_references.py

@@ -16,7 +16,6 @@
 # under the License.
 from __future__ import annotations
 
-import enum
 import os
 import re
 import tempfile
@@ -26,21 +25,14 @@
 
 from rich import print
 
-from airflow_breeze.global_constants import get_available_documentation_provider_packages
+from airflow_breeze.utils.console import get_console
 
 airflow_redirects_link = (
     "https://raw.githubusercontent.com/apache/airflow/main/docs/apache-airflow/redirects.txt"
 )
 helm_redirects_link = "https://raw.githubusercontent.com/apache/airflow/main/docs/helm-chart/redirects.txt"
 
 
-# types of generations supported
-class GenerationType(enum.Enum):
-    airflow = 1
-    helm = 2
-    providers = 3
-
-
 def download_file(url):
     try:
         temp_dir = Path(tempfile.mkdtemp(prefix="temp_dir", suffix=""))
@@ -149,26 +141,23 @@ def generate_back_references(link: str, base_path: Path):
                 create_back_reference_html(relative_path, dest_file_path)
 
 
-def start_generating_back_references(
-    gen_type: GenerationType, airflow_site_directory: Path, short_provider_package_ids: list[str]
-):
-    # Either packages or gen_type should be provided
+def start_generating_back_references(airflow_site_directory: Path, short_provider_package_ids: list[str]):
     docs_archive_path = airflow_site_directory / "docs-archive"
     airflow_docs_path = docs_archive_path / "apache-airflow"
     helm_docs_path = docs_archive_path / "helm-chart"
-
-    if gen_type == GenerationType.airflow:
+    if "apache-airflow" in short_provider_package_ids:
         generate_back_references(airflow_redirects_link, airflow_docs_path)
-    elif gen_type == GenerationType.helm:
+        short_provider_package_ids.remove("apache-airflow")
+    if "helm-chart" in short_provider_package_ids:
         generate_back_references(helm_redirects_link, helm_docs_path)
-    elif gen_type == GenerationType.providers:
-        if short_provider_package_ids:
-            all_providers = [
-                f"apache-airflow-providers-{package.replace('.','-')}"
-                for package in short_provider_package_ids
-            ]
-        else:
-            all_providers = get_available_documentation_provider_packages()
+        short_provider_package_ids.remove("helm-chart")
+    if "docker-stack" in short_provider_package_ids:
+        get_console().print("[info]Skipping docker-stack package. No back-reference needed.")
+        short_provider_package_ids.remove("docker-stack")
+    if short_provider_package_ids:
+        all_providers = [
+            f"apache-airflow-providers-{package.replace('.','-')}" for package in short_provider_package_ids
+        ]
         for p in all_providers:
             print(f"Processing airflow provider: {p}")
             generate_back_references(get_github_redirects_url(p), docs_archive_path / p)

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -44,7 +44,7 @@
     SINGLE_PLATFORMS,
     START_AIRFLOW_ALLOWED_EXECUTORS,
     START_AIRFLOW_DEFAULT_ALLOWED_EXECUTORS,
-    get_available_documentation_provider_packages,
+    get_available_documentation_packages,
 )
 from airflow_breeze.utils.custom_param_types import (
     AnswerChoice,
@@ -443,7 +443,13 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
     "packages",
     nargs=-1,
     required=False,
-    type=BetterChoice(get_available_documentation_provider_packages(short_version=True)),
+    type=BetterChoice(get_available_documentation_packages(short_version=True)),
+)
+argument_packages_plus_all_providers = click.argument(
+    "packages_plus_all_providers",
+    nargs=-1,
+    required=False,
+    type=BetterChoice(["all-providers"] + get_available_documentation_packages(short_version=True)),
 )
 option_airflow_constraints_reference = click.option(
     "--airflow-constraints-reference",

dev/breeze/tests/test_global_constants.py

@@ -16,12 +16,12 @@
 # under the License.
 from __future__ import annotations
 
-from airflow_breeze.global_constants import get_available_documentation_provider_packages
+from airflow_breeze.global_constants import get_available_documentation_packages
 
 AVAILABLE_PACKAGES_STARTING_LIST = ("apache-airflow", "helm-chart", "docker-stack")
 
 
 def test_get_available_packages():
-    assert len(get_available_documentation_provider_packages()) > 70
-    for package in get_available_documentation_provider_packages():
+    assert len(get_available_documentation_packages()) > 70
+    for package in get_available_documentation_packages():
         assert package.startswith(AVAILABLE_PACKAGES_STARTING_LIST)

dev/provider_packages/publish_provider_documentation.sh

@@ -30,4 +30,6 @@ done
 
 breeze build-docs "${provider_filters[@]}"
 
+breeze release-management add-back-references "${@}"
+
 cd "${AIRFLOW_SITE_DIRECTORY}"

images/breeze/output-commands-hash.txt

@@ -2,7 +2,7 @@
 # Please do not solve it but run `breeze setup regenerate-command-images`.
 # This command should fix the conflict and regenerate help images that you have conflict with.
 main:344261ca3aa7ff31e098b1d88280566a
-build-docs:b468701e6c15875fb421a1874fa0c8a2
+build-docs:9b540ef8f8c2bd942647595d1e250cae
 ci:find-backtracking-candidates:17fe56b867a745e5032a08dfcd3f73ee
 ci:fix-ownership:3e5a73533cc96045e72cb258783cfc96
 ci:free-space:49af17b032039c05c41a7a8283f365cc
@@ -35,23 +35,24 @@ k8s:8c1e4287deb0533a74f3b302f9c574be
 prod-image:build:789f32a07099033b49936d4cfbeb5322
 prod-image:pull:76f1f27e6119928412abecf153fce4bb
 prod-image:verify:bd2b78738a7c388dbad6076c41a9f906
-prod-image:7b3369e182724fc155b3399c95d0fd73
-release-management:add-back-references:0d4eb5ed82e5381bc630b343ba605a72
+prod-image:2946a9cee30c8ad7bd738ea3c77bff55
+release-management:add-back-references:9c565c08edd57ab2b484efc2be53672e
 release-management:create-minor-branch:a3834afc4aa5d1e98002c9e9e7a9931d
 release-management:generate-constraints:b8fcaf8f0acd35ed5dbd48659bdb6485
-release-management:generate-issue-content-providers:9e1c3e1072465e79f698ce675a1564cd
+release-management:generate-issue-content-providers:71fccecb5187faf66aaada89f3db9091
 release-management:generate-providers-metadata:d4e8e5cfaa024e3963af02d7a873048d
 release-management:install-provider-packages:7a5aa75736387c442c05163581543cbc
 release-management:prepare-airflow-package:85d01c57e5b5ee0fb9e5f9d9706ed3b5
-release-management:prepare-provider-documentation:cb925862b46fb2fa01cd1e5723d422b1
-release-management:prepare-provider-packages:75fdc618a4a8c37ed4713f9b90020814
-release-management:publish-docs:e40d0fbf7dcc6ca8e2dd4728fc279257
+release-management:prepare-provider-documentation:a87bae187db8bb988f4cace124b23171
+release-management:prepare-provider-packages:cd39b58b8c1b46d6ce26ad53e3577aa4
+release-management:publish-docs:1eb6f4721cd52be497d1623d37170175
 release-management:release-prod-images:cfbfe8b19fee91fd90718f98ef2fd078
 release-management:start-rc-process:b27bd524dd3c89f50a747b60a7e892c1
 release-management:start-release:419f48f6a4ff4457cb9de7ff496aebbe
-release-management:verify-provider-packages:96dce5644aad6b37080acf77b3d8de3a
-release-management:dc4897917210deefb4338f2038f8cb33
-sbom:generate-provider-requirements:9abe53200ea5f40e0bf7c27f6087f27f
+release-management:update-constraints:596d970210f9a71cf6b688995797c0e2
+release-management:verify-provider-packages:2bfa1015b2a4682f7b098587e09026d4
+release-management:394f619fa3803bf148f200c9f0213c5d
+sbom:generate-provider-requirements:230cea8fb70cb4fddd73583d82ca4ede
 sbom:update-sbom-information:0ce56884e5f842e3e80d6619df1ccc64
 sbom:531007d8e9f8415e6c17b7b04dd89b9d
 setup:autocomplete:fffcd49e102e09ccd69b3841a9e3ea8e