Improve handling of environment variables

changelogs/fragments/166-envvars.yml

@@ -0,0 +1,9 @@
+minor_changes:
+  - "Add the roles ``:ansenvvar:`` and ``:ansenvvarref:`` to the antsibull-docs Sphinx extension
+     (https://github.com/ansible-community/antsibull-docs/pull/166)."
+  - "Add the new collection config field ``envvar_directives`` which allows to declare which environment variables are
+     declared with an ``.. envvar::`` directive in the collection's extra docsite documentation. This is used, next to
+     the plugin configuration information and the ansible-core configuration information, to determine whether an environment
+     variable is referencable or not (https://github.com/ansible-community/antsibull-docs/pull/166)."
+  - "Render ``E(...)`` markup with ``:ansenvvarref:`` or ``:ansenvvar:`` depending on whether the environment variable
+     is known to be referencable or not (https://github.com/ansible-community/antsibull-docs/pull/166)."

src/antsibull_docs/cli/doc_commands/_build.py

@@ -23,6 +23,7 @@
     remove_redirect_duplicates,
 )
 from ...env_variables import (
+    collect_referable_envvars,
     collect_referenced_environment_variables,
     load_ansible_config,
 )
@@ -160,9 +161,12 @@ def generate_docs_for_all_collections(
 
     # Handle environment variables
     ansible_config = load_ansible_config(full_collection_metadata["ansible.builtin"])
-    referenced_env_vars = collect_referenced_environment_variables(
+    referenced_env_vars, core_env_vars = collect_referenced_environment_variables(
         new_plugin_info, ansible_config
     )
+    referable_envvars = collect_referable_envvars(
+        referenced_env_vars, core_env_vars, collection_metadata
+    )
 
     collection_namespaces = get_collection_namespaces(collection_to_plugin_info.keys())
 
@@ -185,6 +189,7 @@ def generate_docs_for_all_collections(
                 collection_install=collection_install,
                 breadcrumbs=breadcrumbs,
                 for_official_docsite=for_official_docsite,
+                referable_envvars=referable_envvars,
             )
         )
         flog.notice("Finished writing collection index")
@@ -196,6 +201,7 @@ def generate_docs_for_all_collections(
                 collection_install=collection_install,
                 breadcrumbs=breadcrumbs,
                 for_official_docsite=for_official_docsite,
+                referable_envvars=referable_envvars,
             )
         )
         flog.notice("Finished writing collection namespace index")
@@ -207,6 +213,7 @@ def generate_docs_for_all_collections(
                 collection_url=collection_url,
                 collection_install=collection_install,
                 for_official_docsite=for_official_docsite,
+                referable_envvars=referable_envvars,
             )
         )
         flog.notice("Finished writing plugin indexes")
@@ -217,6 +224,7 @@ def generate_docs_for_all_collections(
                 collection_url=collection_url,
                 collection_install=collection_install,
                 for_official_docsite=for_official_docsite,
+                referable_envvars=referable_envvars,
             )
         )
         flog.notice("Finished writing callback plugin indexes")
@@ -233,6 +241,7 @@ def generate_docs_for_all_collections(
             link_data=link_data,
             breadcrumbs=breadcrumbs,
             for_official_docsite=for_official_docsite,
+            referable_envvars=referable_envvars,
         )
     )
     flog.notice("Finished writing indexes")
@@ -247,6 +256,7 @@ def generate_docs_for_all_collections(
             link_data=link_data,
             squash_hierarchy=squash_hierarchy,
             for_official_docsite=for_official_docsite,
+            referable_envvars=referable_envvars,
         )
     )
     flog.debug("Finished writing plugin stubs")
@@ -264,6 +274,7 @@ def generate_docs_for_all_collections(
             squash_hierarchy=squash_hierarchy,
             use_html_blobs=use_html_blobs,
             for_official_docsite=for_official_docsite,
+            referable_envvars=referable_envvars,
         )
     )
     flog.debug("Finished writing plugin docs")
@@ -275,7 +286,10 @@ def generate_docs_for_all_collections(
 
     asyncio.run(
         output_environment_variables(
-            dest_dir, referenced_env_vars, squash_hierarchy=squash_hierarchy
+            dest_dir,
+            referenced_env_vars,
+            squash_hierarchy=squash_hierarchy,
+            referable_envvars=referable_envvars,
         )
     )
     flog.debug("Finished writing environment variables")

src/antsibull_docs/env_variables.py

@@ -114,14 +114,17 @@ def _augment_env_var_descriptions(
 def collect_referenced_environment_variables(
     plugin_info: Mapping[str, Mapping[str, t.Any]],
     ansible_config: Mapping[str, Mapping[str, t.Any]],
-) -> Mapping[str, EnvironmentVariableInfo]:
+) -> tuple[Mapping[str, EnvironmentVariableInfo], set[str]]:
     """
     Collect referenced environment variables that are not defined in the ansible-core
     configuration.
 
     :arg plugin_info: Mapping of plugin type to a mapping of plugin name to plugin record.
     :arg ansible_config: The Ansible base configuration (``lib/ansible/config/base.yml``).
-    :returns: A Mapping of environment variable name to an environment variable infomation object.
+    :returns: A tuple consisting of a
+        Mapping of environment variable name to an environment variable infomation object,
+        and a set of environment variable names that are part of the ansible-core
+        configuration.
     """
     core_envs = {"ANSIBLE_CONFIG"}
     for config in ansible_config.values():
@@ -133,4 +136,26 @@ def collect_referenced_environment_variables(
         plugin_info, core_envs
     )
     _augment_env_var_descriptions(other_variables, other_variable_description)
-    return other_variables
+    return other_variables, core_envs
+
+
+def collect_referable_envvars(
+    referenced_env_vars: Mapping[str, EnvironmentVariableInfo],
+    core_env_vars: set[str],
+    collection_metadata: Mapping[str, AnsibleCollectionMetadata],
+) -> set[str]:
+    """
+    :arg referenced_env_vars: A Mapping of environment variable name to an
+        environment variable infomation object.
+    :arg core_env_vars: Set of environment variable names that are part of the
+        ansible-core configuration.
+    :arg collection_metadata: A Mapping of collection names to collection metadata
+        objects.
+    :returns: A set of environment variables that can be referenced via the
+        ``:envvar:`` role.
+    """
+    referable_envvars = set(referenced_env_vars)
+    referable_envvars.update(core_env_vars)
+    for collection_meta in collection_metadata.values():
+        referable_envvars.update(collection_meta.docs_config.envvar_directives)
+    return referable_envvars

src/antsibull_docs/jinja2/environment.py

@@ -60,6 +60,7 @@ def doc_environment(
     extra_tests: Mapping[str, t.Callable] | None = None,
     collection_url: CollectionNameTransformer | None = None,
     collection_install: CollectionNameTransformer | None = None,
+    referable_envvars: set[str] | None = None,
 ) -> Environment:
     loader: BaseLoader
     if isinstance(template_location, str) and os.path.exists(template_location):
@@ -86,7 +87,6 @@ def doc_environment(
     # with <Jinja-2.10
     env.globals["to_kludge_ns"] = to_kludge_ns
     env.globals["from_kludge_ns"] = from_kludge_ns
-    env.globals["reference_plugin_rst"] = reference_plugin_rst
     if "max" not in env.filters:
         # Jinja < 2.10
         env.filters["max"] = do_max
@@ -95,6 +95,8 @@ def doc_environment(
         # Jinja < 2.9
         env.filters["tojson"] = json.dumps
 
+    env.globals["reference_plugin_rst"] = reference_plugin_rst
+    env.globals["referable_envvars"] = referable_envvars
     env.filters["rst_ify"] = rst_ify
     env.filters["html_ify"] = html_ify
     env.filters["fmt"] = rst_fmt

src/antsibull_docs/jinja2/filters.py

@@ -161,6 +161,7 @@ def rst_ify(
         plugin_fqcn=plugin_fqcn,
         plugin_type=plugin_type,
         role_entrypoint=role_entrypoint,
+        referable_envvars=context.get("referable_envvars"),
     )
 
     flog.fields(counts=counts).info("Number of macros converted to rst equivalents")

src/antsibull_docs/lint_plugin_docs.py

@@ -673,6 +673,7 @@ def _lint_collection_plugin_docs(
         ("antsibull_docs.data", "docsite"),
         collection_url=collection_url,
         collection_install=collection_install,
+        referable_envvars=None,  # this shouldn't make a difference for validation
     )
     # Get the templates
     plugin_tmpl = env.get_template("plugin.rst.j2")

src/antsibull_docs/markup/rstify.py

@@ -13,6 +13,7 @@
 
 from antsibull_docs_parser import dom
 from antsibull_docs_parser.parser import Context, parse
+from antsibull_docs_parser.rst import AntsibullRSTFormatter
 from antsibull_docs_parser.rst import rst_escape as _rst_escape
 from antsibull_docs_parser.rst import to_rst
 
@@ -31,7 +32,18 @@ def rst_code(value: str) -> str:
     """Write value as :code:`...` RST construct."""
     if not isinstance(value, str):
         value = str(value)
-    return f":code:`{rst_escape(value, escape_ending_whitespace=True)}`"
+    return f":code:`{_rst_escape(value, escape_ending_whitespace=True)}`"
+
+
+class CustomizedAntsibullRSTFormatter(AntsibullRSTFormatter):
+    def __init__(self, referable_envvars: set[str] | None = None):
+        self._referable_envvars = referable_envvars or set()
+
+    def format_env_variable(self, part: dom.EnvVariablePart) -> str:
+        envvar = part.name.split("=", 1)[0].strip()
+        if envvar in self._referable_envvars:
+            return f"\\ :ansenvvarref:`{_rst_escape(part.name, True)}`\\ "
+        return f"\\ :ansenvvar:`{_rst_escape(part.name, True)}`\\ "
 
 
 def rst_ify(
@@ -40,13 +52,18 @@ def rst_ify(
     plugin_fqcn: str | None = None,
     plugin_type: str | None = None,
     role_entrypoint: str | None = None,
+    referable_envvars: set[str] | None = None,
 ) -> tuple[str, Mapping[str, int]]:
     """convert symbols like I(this is in italics) to valid restructured text"""
     current_plugin: dom.PluginIdentifier | None = None
     if plugin_fqcn and plugin_type:
         current_plugin = dom.PluginIdentifier(fqcn=plugin_fqcn, type=plugin_type)
     context = Context(current_plugin=current_plugin, role_entrypoint=role_entrypoint)
     paragraphs = parse(text, context, errors="message")
-    text = to_rst(paragraphs, current_plugin=current_plugin)
+    text = to_rst(
+        paragraphs,
+        current_plugin=current_plugin,
+        formatter=CustomizedAntsibullRSTFormatter(referable_envvars),
+    )
     counts = _count(paragraphs)
     return text, counts

src/antsibull_docs/schemas/collection_config.py

@@ -13,4 +13,10 @@
 
 
 class CollectionConfig(p.BaseModel):
+    # Whether the collection uses flatmapping to flatten subdirectories in
+    # `plugins/*/`.
     flatmap: bool = False
+
+    # List of environment variables that are defined by `.. envvar::` directives
+    # in the extra docsite RST files.
+    envvar_directives: list[str] = []

src/antsibull_docs/write_docs/collections.py

@@ -137,6 +137,7 @@ async def output_indexes(
     squash_hierarchy: bool = False,
     breadcrumbs: bool = True,
     for_official_docsite: bool = False,
+    referable_envvars: set[str] | None = None,
 ) -> None:
     """
     Generate collection-level index pages for the collections.
@@ -153,6 +154,7 @@ async def output_indexes(
         disabled.  This will disable breadcrumbs but save on memory usage.
     :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
         official docsite on docs.ansible.com.
+    :kwarg referable_envvars: Optional set of environment variables that can be referenced.
     """
     flog = mlog.fields(func="output_indexes")
     flog.debug("Enter")
@@ -164,6 +166,7 @@ async def output_indexes(
         ("antsibull_docs.data", "docsite"),
         collection_url=collection_url,
         collection_install=collection_install,
+        referable_envvars=referable_envvars,
     )
     # Get the templates
     collection_plugins_tmpl = env.get_template("plugins_by_collection.rst.j2")

src/antsibull_docs/write_docs/hierarchy.py

@@ -103,6 +103,7 @@ async def output_collection_index(
     collection_install: CollectionNameTransformer,
     breadcrumbs: bool = True,
     for_official_docsite: bool = False,
+    referable_envvars: set[str] | None = None,
 ) -> None:
     """
     Generate top-level collection index page for the collections.
@@ -115,6 +116,7 @@ async def output_collection_index(
         disabled.  This will disable breadcrumbs but save on memory usage.
     :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
         official docsite on docs.ansible.com.
+    :kwarg referable_envvars: Optional set of environment variables that can be referenced.
     """
     flog = mlog.fields(func="output_collection_index")
     flog.debug("Enter")
@@ -123,6 +125,7 @@ async def output_collection_index(
         ("antsibull_docs.data", "docsite"),
         collection_url=collection_url,
         collection_install=collection_install,
+        referable_envvars=referable_envvars,
     )
     # Get the templates
     collection_list_tmpl = env.get_template("list_of_collections.rst.j2")
@@ -154,6 +157,7 @@ async def output_collection_namespace_indexes(
     collection_install: CollectionNameTransformer,
     breadcrumbs: bool = True,
     for_official_docsite: bool = False,
+    referable_envvars: set[str] | None = None,
 ) -> None:
     """
     Generate collection namespace index pages for the collections.
@@ -164,6 +168,7 @@ async def output_collection_namespace_indexes(
         disabled.  This will disable breadcrumbs but save on memory usage.
     :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
         official docsite on docs.ansible.com.
+    :kwarg referable_envvars: Optional set of environment variables that can be referenced.
     """
     flog = mlog.fields(func="output_collection_namespace_indexes")
     flog.debug("Enter")
@@ -172,6 +177,7 @@ async def output_collection_namespace_indexes(
         ("antsibull_docs.data", "docsite"),
         collection_url=collection_url,
         collection_install=collection_install,
+        referable_envvars=referable_envvars,
     )
     # Get the templates
     collection_list_tmpl = env.get_template("list_of_collections_by_namespace.rst.j2")

src/antsibull_docs/write_docs/indexes.py

@@ -94,6 +94,7 @@ async def output_callback_indexes(
     collection_url: CollectionNameTransformer,
     collection_install: CollectionNameTransformer,
     for_official_docsite: bool = False,
+    referable_envvars: set[str] | None = None,
 ) -> None:
     """
     Generate top-level callback plugin index pages for all callback plugins of a type in all
@@ -104,6 +105,7 @@ async def output_callback_indexes(
     :arg dest_dir: The directory to place the documentation in.
     :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
         official docsite on docs.ansible.com.
+    :kwarg referable_envvars: Optional set of environment variables that can be referenced.
     """
     flog = mlog.fields(func="output_callback_indexes")
     flog.debug("Enter")
@@ -112,6 +114,7 @@ async def output_callback_indexes(
         ("antsibull_docs.data", "docsite"),
         collection_url=collection_url,
         collection_install=collection_install,
+        referable_envvars=referable_envvars,
     )
     # Get the templates
     plugin_list_tmpl = env.get_template("list_of_callback_plugins.rst.j2")
@@ -155,6 +158,7 @@ async def output_plugin_indexes(
     collection_url: CollectionNameTransformer,
     collection_install: CollectionNameTransformer,
     for_official_docsite: bool = False,
+    referable_envvars: set[str] | None = None,
 ) -> None:
     """
     Generate top-level plugin index pages for all plugins of a type in all collections.
@@ -165,6 +169,7 @@ async def output_plugin_indexes(
     :arg dest_dir: The directory to place the documentation in.
     :kwarg for_official_docsite: Default False.  Set to True to use wording specific for the
         official docsite on docs.ansible.com.
+    :kwarg referable_envvars: Optional set of environment variables that can be referenced.
     """
     flog = mlog.fields(func="output_plugin_indexes")
     flog.debug("Enter")
@@ -173,6 +178,7 @@ async def output_plugin_indexes(
         ("antsibull_docs.data", "docsite"),
         collection_url=collection_url,
         collection_install=collection_install,
+        referable_envvars=referable_envvars,
     )
     # Get the templates
     plugin_list_tmpl = env.get_template("list_of_plugins.rst.j2")
@@ -212,6 +218,7 @@ async def output_environment_variables(
     dest_dir: str,
     env_variables: Mapping[str, EnvironmentVariableInfo],
     squash_hierarchy: bool = False,
+    referable_envvars: set[str] | None = None,
 ) -> None:
     """
     Write environment variable Generate collection-level index pages for the collections.
@@ -221,6 +228,7 @@ async def output_environment_variables(
     :arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used.
                            Undefined behavior if documentation for multiple collections are
                            created.
+    :kwarg referable_envvars: Optional set of environment variables that can be referenced.
     """
     flog = mlog.fields(func="write_environment_variables")
     flog.debug("Enter")
@@ -230,7 +238,10 @@ async def output_environment_variables(
     else:
         collection_toplevel = dest_dir
 
-    env = doc_environment(("antsibull_docs.data", "docsite"))
+    env = doc_environment(
+        ("antsibull_docs.data", "docsite"),
+        referable_envvars=referable_envvars,
+    )
     # Get the templates
     env_var_list_tmpl = env.get_template("list_of_env_variables.rst.j2")