diff --git a/changelogs/fragments/12-lint-collection-docs-plugin-rst.yml b/changelogs/fragments/12-lint-collection-docs-plugin-rst.yml new file mode 100644 index 00000000..1e57d663 --- /dev/null +++ b/changelogs/fragments/12-lint-collection-docs-plugin-rst.yml @@ -0,0 +1,4 @@ +minor_changes: + - "The ``lint-collection-docs`` subcommand has a new boolean flag ``--plugin-docs`` which renders the plugin docs + to RST and validates them with rstcheck. This can be used as a lighter version of rendering the docsite in CI + (https://github.com/ansible-community/antsibull-docs/pull/12)." diff --git a/src/antsibull_docs/cli/antsibull_docs.py b/src/antsibull_docs/cli/antsibull_docs.py index b86cc4d2..72e9faef 100644 --- a/src/antsibull_docs/cli/antsibull_docs.py +++ b/src/antsibull_docs/cli/antsibull_docs.py @@ -360,6 +360,12 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: metavar='/path/to/collection', help='path to collection (directory that includes' ' galaxy.yml)') + lint_collection_docs_parser.add_argument('--plugin-docs', + dest='plugin_docs', action=BooleanOptionalAction, + default=False, + help='Determine whether to also check RST file' + ' generation and validation for plugins and roles' + ' in this collection. (default: True)') flog.debug('Argument parser setup') diff --git a/src/antsibull_docs/cli/doc_commands/lint_collection_docs.py b/src/antsibull_docs/cli/doc_commands/lint_collection_docs.py index a29fa5dd..a043c9e4 100644 --- a/src/antsibull_docs/cli/doc_commands/lint_collection_docs.py +++ b/src/antsibull_docs/cli/doc_commands/lint_collection_docs.py @@ -8,6 +8,7 @@ from ...collection_links import lint_collection_links from ...lint_extra_docs import lint_collection_extra_docs_files +from ...lint_plugin_docs import lint_collection_plugin_docs mlog = log.fields(mod=__name__) @@ -26,6 +27,7 @@ def lint_collection_docs() -> int: app_ctx = app_context.app_ctx.get() collection_root = app_ctx.extra['collection_root_path'] + plugin_docs = app_ctx.extra['plugin_docs'] flog.notice('Linting extra docs files') errors = lint_collection_extra_docs_files(collection_root) @@ -33,6 +35,10 @@ def lint_collection_docs() -> int: flog.notice('Linting collection links') errors.extend(lint_collection_links(collection_root)) + if plugin_docs: + flog.notice('Linting plugin docs') + errors.extend(lint_collection_plugin_docs(collection_root)) + messages = sorted(set(f'{error[0]}:{error[1]}:{error[2]}: {error[3]}' for error in errors)) for message in messages: diff --git a/src/antsibull_docs/docs_parsing/ansible_doc.py b/src/antsibull_docs/docs_parsing/ansible_doc.py index 9b017584..4ae27525 100644 --- a/src/antsibull_docs/docs_parsing/ansible_doc.py +++ b/src/antsibull_docs/docs_parsing/ansible_doc.py @@ -184,6 +184,31 @@ def _extract_ansible_builtin_metadata(stdout: str) -> AnsibleCollectionMetadata: return AnsibleCollectionMetadata(path=path, version=version) +def parse_ansible_galaxy_collection_list(raw_output: str, + collection_names: t.Optional[t.List[str]] = None, + ) -> t.List[t.Tuple[str, str, str, t.Optional[str]]]: + result = [] + current_base_path = None + for line in raw_output.splitlines(): + parts = line.split() + if len(parts) >= 2: + if parts[0] == '#': + current_base_path = parts[1] + elif current_base_path is not None: + collection_name = parts[0] + version = parts[1] + if '.' in collection_name: + if collection_names is None or collection_name in collection_names: + namespace, name = collection_name.split('.', 2) + result.append(( + namespace, + name, + os.path.join(current_base_path, namespace, name), + None if version == '*' else version + )) + return result + + def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'], env: t.Dict[str, str], collection_names: t.Optional[t.List[str]] = None, @@ -201,21 +226,9 @@ def get_collection_metadata(venv: t.Union['VenvRunner', 'FakeVenvRunner'], venv_ansible_galaxy = venv.get_command('ansible-galaxy') ansible_collection_list_cmd = venv_ansible_galaxy('collection', 'list', _env=env) raw_result = ansible_collection_list_cmd.stdout.decode('utf-8', errors='surrogateescape') - current_base_path = None - for line in raw_result.splitlines(): - parts = line.split() - if len(parts) >= 2: - if parts[0] == '#': - current_base_path = parts[1] - else: - collection_name = parts[0] - version = parts[1] - if '.' in collection_name: - if collection_names is None or collection_name in collection_names: - namespace, name = collection_name.split('.', 2) - collection_metadata[collection_name] = AnsibleCollectionMetadata( - path=os.path.join(current_base_path, namespace, name), - version=None if version == '*' else version) + for namespace, name, path, version in parse_ansible_galaxy_collection_list(raw_result): + collection_metadata[f'{namespace}.{name}'] = AnsibleCollectionMetadata( + path=path, version=version) return collection_metadata diff --git a/src/antsibull_docs/lint_extra_docs.py b/src/antsibull_docs/lint_extra_docs.py index 0b47fae5..c9917805 100644 --- a/src/antsibull_docs/lint_extra_docs.py +++ b/src/antsibull_docs/lint_extra_docs.py @@ -1,17 +1,14 @@ # coding: utf-8 -# Author: Felix Fontein +# Author: Felix Fontein # License: GPLv3+ # Copyright: Ansible Project, 2021 """Lint extra collection documentation in docs/docsite/.""" -import json import os import os.path import re import typing as t -from antsibull_core.yaml import load_yaml_file - from .extra_docs import ( find_extra_docs, lint_required_conditions, @@ -20,28 +17,12 @@ ) from .rstcheck import check_rst_content +from .lint_helpers import ( + load_collection_name, +) -_RST_LABEL_DEFINITION = re.compile(r'''^\.\. _([^:]+):''') - - -def load_collection_name(path_to_collection: str) -> str: - '''Load collection name (namespace.name) from collection's galaxy.yml.''' - manifest_json_path = os.path.join(path_to_collection, 'MANIFEST.json') - if os.path.isfile(manifest_json_path): - with open(manifest_json_path, 'rb') as f: - manifest_json = json.load(f) - # pylint:disable-next=consider-using-f-string - collection_name = '{namespace}.{name}'.format(**manifest_json['collection_info']) - return collection_name - - galaxy_yml_path = os.path.join(path_to_collection, 'galaxy.yml') - if os.path.isfile(galaxy_yml_path): - galaxy_yml = load_yaml_file(galaxy_yml_path) - # pylint:disable-next=consider-using-f-string - collection_name = '{namespace}.{name}'.format(**galaxy_yml) - return collection_name - raise Exception(f'Cannot find files {manifest_json_path} and {galaxy_yml_path}') +_RST_LABEL_DEFINITION = re.compile(r'''^\.\. _([^:]+):''') # pylint:disable-next=unused-argument diff --git a/src/antsibull_docs/lint_helpers.py b/src/antsibull_docs/lint_helpers.py new file mode 100644 index 00000000..1949b40b --- /dev/null +++ b/src/antsibull_docs/lint_helpers.py @@ -0,0 +1,36 @@ +# coding: utf-8 +# Author: Felix Fontein +# License: GPLv3+ +# Copyright: Ansible Project, 2022 +"""Lint plugin docs.""" + +import json +import os +import os.path +import typing as t + +from antsibull_core.yaml import load_yaml_file + + +def load_collection_info(path_to_collection: str) -> t.Dict[str, t.Any]: + '''Load collection name (namespace.name) from collection's galaxy.yml.''' + manifest_json_path = os.path.join(path_to_collection, 'MANIFEST.json') + if os.path.isfile(manifest_json_path): + with open(manifest_json_path, 'rb') as f: + manifest_json = json.load(f) + return manifest_json['collection_info'] + + galaxy_yml_path = os.path.join(path_to_collection, 'galaxy.yml') + if os.path.isfile(galaxy_yml_path): + galaxy_yml = load_yaml_file(galaxy_yml_path) + return galaxy_yml + + raise Exception(f'Cannot find files {manifest_json_path} and {galaxy_yml_path}') + + +def load_collection_name(path_to_collection: str) -> str: + '''Load collection name (namespace.name) from collection's galaxy.yml.''' + info = load_collection_info(path_to_collection) + # pylint:disable-next=consider-using-f-string + collection_name = '{namespace}.{name}'.format(**info) + return collection_name diff --git a/src/antsibull_docs/lint_plugin_docs.py b/src/antsibull_docs/lint_plugin_docs.py new file mode 100644 index 00000000..7e352e3f --- /dev/null +++ b/src/antsibull_docs/lint_plugin_docs.py @@ -0,0 +1,191 @@ +# coding: utf-8 +# Author: Felix Fontein +# License: GPLv3+ +# Copyright: Ansible Project, 2022 +"""Lint plugin docs.""" + +import os +import shutil +import tempfile +import typing as t + +import sh + +from antsibull_core.compat import asyncio_run +from antsibull_core.venv import FakeVenvRunner + +from .lint_helpers import ( + load_collection_info, +) + +from .docs_parsing.ansible_doc import ( + parse_ansible_galaxy_collection_list, +) + +from .augment_docs import augment_docs +from .cli.doc_commands.stable import ( + normalize_all_plugin_info, + get_plugin_contents, + get_collection_contents, +) +from .collection_links import load_collections_links +from .docs_parsing.parsing import get_ansible_plugin_info +from .docs_parsing.routing import ( + load_all_collection_routing, + remove_redirect_duplicates, +) +from .jinja2.environment import doc_environment +from .write_docs import create_plugin_rst +from .rstcheck import check_rst_content + + +class CollectionCopier: + dir: t.Optional[str] + + def __init__(self): + self.dir = None + + def __enter__(self): + if self.dir is None: + raise AssertionError('Collection copier already initialized') + self.dir = os.path.realpath(tempfile.mkdtemp(prefix='antsibull-docs-')) + return self + + def add_collection(self, collecion_source_path: str, namespace: str, name: str) -> None: + self_dir = self.dir + if self_dir is None: + raise AssertionError('Collection copier not initialized') + collection_container_dir = os.path.join( + self_dir, 'ansible_collections', namespace) + os.makedirs(collection_container_dir, exist_ok=True) + + collection_dir = os.path.join(collection_container_dir, name) + shutil.copytree(collecion_source_path, collection_dir, symlinks=True) + + def __exit__(self, type_, value, traceback_): + self_dir = self.dir + if self_dir is None: + raise AssertionError('Collection copier not initialized') + shutil.rmtree(self_dir, ignore_errors=True) + self.dir = None + + +class CollectionFinder: + def __init__(self): + self.collections = {} + stdout = sh.Command('ansible-galaxy')('collection', 'list').stdout + raw_output = stdout.decode('utf-8', errors='surrogateescape') + for namespace, name, path, _ in reversed(parse_ansible_galaxy_collection_list(raw_output)): + self.collections[f'{namespace}.{name}'] = path + + def find(self, namespace, name): + return self.collections.get(f'{namespace}.{name}') + + +def _lint_collection_plugin_docs(collections_dir: str, collection_name: str, + original_path_to_collection: str, + ) -> t.List[t.Tuple[str, int, int, str]]: + # Load collection docs + venv = FakeVenvRunner() + plugin_info, collection_metadata = asyncio_run(get_ansible_plugin_info( + venv, collections_dir, collection_names=[collection_name])) + # Load routing information + collection_routing = asyncio_run(load_all_collection_routing(collection_metadata)) + # Process data + remove_redirect_duplicates(plugin_info, collection_routing) + plugin_info, nonfatal_errors = asyncio_run(normalize_all_plugin_info(plugin_info)) + augment_docs(plugin_info) + # Load link data + link_data = asyncio_run(load_collections_links( + {name: data.path for name, data in collection_metadata.items()})) + # More processing + plugin_contents = get_plugin_contents(plugin_info, nonfatal_errors) + collection_to_plugin_info = get_collection_contents(plugin_contents) + for collection in collection_metadata: + collection_to_plugin_info[collection] # pylint:disable=pointless-statement + # Collect non-fatal errors + result = [] + for plugin_type, plugins in sorted(nonfatal_errors.items()): + for plugin_name, errors in sorted(plugins.items()): + for error in errors: + result.append(( + os.path.join(original_path_to_collection, 'plugins', plugin_type, plugin_name), + 0, + 0, + error, + )) + # Compose RST files and check for errors + # Setup the jinja environment + env = doc_environment(('antsibull_docs.data', 'docsite')) + # Get the templates + plugin_tmpl = env.get_template('plugin.rst.j2') + role_tmpl = env.get_template('role.rst.j2') + error_tmpl = env.get_template('plugin-error.rst.j2') + + for collection_name_, plugins_by_type in collection_to_plugin_info.items(): + for plugin_type, plugins in plugins_by_type.items(): + plugin_type_tmpl = plugin_tmpl + if plugin_type == 'role': + plugin_type_tmpl = role_tmpl + for plugin_short_name, dummy_ in plugins.items(): + plugin_name = '.'.join((collection_name_, plugin_short_name)) + rst_content = create_plugin_rst( + collection_name_, collection_metadata[collection_name_], + link_data[collection_name_], + plugin_short_name, plugin_type, + plugin_info[plugin_type].get(plugin_name), + nonfatal_errors[plugin_type][plugin_name], + plugin_type_tmpl, error_tmpl, + use_html_blobs=False, + ) + path = os.path.join( + original_path_to_collection, 'plugins', plugin_type, + f'{plugin_short_name}.rst') + rst_results = check_rst_content( + rst_content, filename=path, + ignore_directives=['rst-class'], + ) + result.extend([(path, result[0], result[1], result[2]) for result in rst_results]) + return result + + +def lint_collection_plugin_docs(path_to_collection: str) -> t.List[t.Tuple[str, int, int, str]]: + try: + info = load_collection_info(path_to_collection) + namespace = info['namespace'] + name = info['name'] + dependencies = info.get('dependencies') or {} + except Exception: # pylint:disable=broad-except + return [( + path_to_collection, 0, 0, + 'Cannot identify collection with galaxy.yml or MANIFEST.json at this path')] + result = [] + collection_name = f'{namespace}.{name}' + done_dependencies = {collection_name} + dependencies = sorted(dependencies) + with CollectionCopier() as copier: + # Copy collection + copier.add_collection(path_to_collection, namespace, name) + # Copy all dependencies + if dependencies: + collection_finder = CollectionFinder() + while dependencies: + dependency = dependencies.pop(0) + if dependency in done_dependencies: + continue + dep_namespace, dep_name = dependency.split('.', 2) + dep_collection_path = collection_finder.find(dep_namespace, dep_name) + if dep_collection_path: + copier.add_collection(dep_collection_path, dep_namespace, dep_name) + try: + info = load_collection_info(dep_collection_path) + dependencies.extend(sorted(info.get('dependencies') or {})) + except Exception: # pylint:disable=broad-except + result.append(( + dep_collection_path, 0, 0, + 'Cannot identify collection with galaxy.yml or MANIFEST.json' + ' at this path')) + # Load docs + result.extend(_lint_collection_plugin_docs( + copier.dir, collection_name, path_to_collection)) + return result diff --git a/src/antsibull_docs/rstcheck.py b/src/antsibull_docs/rstcheck.py index c5f1bb1c..2c19a415 100644 --- a/src/antsibull_docs/rstcheck.py +++ b/src/antsibull_docs/rstcheck.py @@ -20,7 +20,9 @@ import rstcheck -def check_rst_content(content: str, filename: t.Optional[str] = None +def check_rst_content(content: str, filename: t.Optional[str] = None, + ignore_directives: t.Optional[t.List[str]] = None, + ignore_roles: t.Optional[t.List[str]] = None, ) -> t.List[t.Tuple[int, int, str]]: ''' Check the content with rstcheck. Return list of errors and warnings. @@ -36,10 +38,14 @@ def check_rst_content(content: str, filename: t.Optional[str] = None f.write(content) config = rstcheck_core.config.RstcheckConfig( report_level=rstcheck_core.config.ReportLevel.WARNING, + ignore_directives=ignore_directives, + ignore_roles=ignore_roles, ) core_results = rstcheck_core.checker.check_file(pathlib.Path(rst_path), config) return [(result.line_number, 0, result.message) for result in core_results] else: + if ignore_directives or ignore_roles: + rstcheck.ignore_directives_and_roles(ignore_directives or [], ignore_roles or []) results = rstcheck.check( content, filename=filename, diff --git a/src/antsibull_docs/write_docs.py b/src/antsibull_docs/write_docs.py index 801cea53..0eea0626 100644 --- a/src/antsibull_docs/write_docs.py +++ b/src/antsibull_docs/write_docs.py @@ -51,7 +51,7 @@ def follow_relative_links(path: str) -> str: :arg path: Path to a file. """ - flog = mlog.fields(func='write_plugin_rst') + flog = mlog.fields(func='follow_relative_links') flog.fields(path=path).debug('Enter') original_path = path @@ -90,17 +90,15 @@ def follow_relative_links(path: str) -> str: path = os.path.join(os.path.dirname(path), link) -async def write_plugin_rst(collection_name: str, - collection_meta: AnsibleCollectionMetadata, - collection_links: CollectionLinks, - plugin_short_name: str, plugin_type: str, - plugin_record: t.Dict[str, t.Any], nonfatal_errors: t.Sequence[str], - plugin_tmpl: Template, error_tmpl: Template, dest_dir: str, - path_override: t.Optional[str] = None, - squash_hierarchy: bool = False, - use_html_blobs: bool = False) -> None: +def create_plugin_rst(collection_name: str, + collection_meta: AnsibleCollectionMetadata, + collection_links: CollectionLinks, + plugin_short_name: str, plugin_type: str, + plugin_record: t.Dict[str, t.Any], nonfatal_errors: t.Sequence[str], + plugin_tmpl: Template, error_tmpl: Template, + use_html_blobs: bool = False) -> str: """ - Write the rst page for one plugin. + Create the rst page for one plugin. :arg collection_name: Dotted colection name. :arg collection_meta: Collection metadata object. @@ -113,19 +111,12 @@ async def write_plugin_rst(collection_name: str, of some or all of the docs :arg plugin_tmpl: Template for the plugin. :arg error_tmpl: Template to use when there wasn't enough documentation for the plugin. - :arg dest_dir: Destination directory for the plugin data. For instance, - :file:`ansible-checkout/docs/docsite/rst/`. The directory structure underneath this - directory will be created if needed. - :arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used. - Undefined behavior if documentation for multiple collections are - created. :arg use_html_blobs: If set to ``True``, will use HTML blobs for parameter and return value tables instead of using RST tables. """ - flog = mlog.fields(func='write_plugin_rst') + flog = mlog.fields(func='create_plugin_rst') flog.debug('Enter') - namespace, collection = collection_name.split('.') plugin_name = '.'.join((collection_name, plugin_short_name)) edit_on_github_url = None @@ -212,6 +203,60 @@ async def write_plugin_rst(collection_name: str, collection_communication=collection_links.communication, ) + flog.debug('Leave') + return plugin_contents + + +async def write_plugin_rst(collection_name: str, + collection_meta: AnsibleCollectionMetadata, + collection_links: CollectionLinks, + plugin_short_name: str, plugin_type: str, + plugin_record: t.Dict[str, t.Any], nonfatal_errors: t.Sequence[str], + plugin_tmpl: Template, error_tmpl: Template, dest_dir: str, + path_override: t.Optional[str] = None, + squash_hierarchy: bool = False, + use_html_blobs: bool = False) -> None: + """ + Write the rst page for one plugin. + + :arg collection_name: Dotted colection name. + :arg collection_meta: Collection metadata object. + :arg collection_links: Collection links object. + :arg plugin_short_name: short name for the plugin. + :arg plugin_type: The type of the plugin. (module, inventory, etc) + :arg plugin_record: The record for the plugin. doc, examples, and return are the + toplevel fields. + :arg nonfatal_errors: Mapping of plugin to any nonfatal errors that will be displayed in place + of some or all of the docs + :arg plugin_tmpl: Template for the plugin. + :arg error_tmpl: Template to use when there wasn't enough documentation for the plugin. + :arg dest_dir: Destination directory for the plugin data. For instance, + :file:`ansible-checkout/docs/docsite/rst/`. The directory structure underneath this + directory will be created if needed. + :arg squash_hierarchy: If set to ``True``, no directory hierarchy will be used. + Undefined behavior if documentation for multiple collections are + created. + :arg use_html_blobs: If set to ``True``, will use HTML blobs for parameter and return value + tables instead of using RST tables. + """ + flog = mlog.fields(func='write_plugin_rst') + flog.debug('Enter') + + namespace, collection = collection_name.split('.') + + plugin_contents = create_plugin_rst( + collection_name=collection_name, + collection_meta=collection_meta, + collection_links=collection_links, + plugin_short_name=plugin_short_name, + plugin_type=plugin_type, + plugin_record=plugin_record, + nonfatal_errors=nonfatal_errors, + plugin_tmpl=plugin_tmpl, + error_tmpl=error_tmpl, + use_html_blobs=use_html_blobs, + ) + if path_override is not None: plugin_file = path_override else: diff --git a/stubs/rstcheck.pyi b/stubs/rstcheck.pyi index 5eec139f..48b950de 100644 --- a/stubs/rstcheck.pyi +++ b/stubs/rstcheck.pyi @@ -8,3 +8,5 @@ def check(source: str, report_level: Union[docutils.utils.Reporter, int] = ..., ignore: Union[dict, None] = ..., debug: bool = ...) -> List[Tuple[int, str]]: ... + +def ignore_directives_and_roles(directives: List[str], roles: List[str]) -> None: ... diff --git a/stubs/rstcheck_core/config.pyi b/stubs/rstcheck_core/config.pyi index 3d952999..064bac46 100644 --- a/stubs/rstcheck_core/config.pyi +++ b/stubs/rstcheck_core/config.pyi @@ -12,4 +12,11 @@ class ReportLevel(enum.Enum): class RstcheckConfig: - def __init__(self, report_level: Optional[ReportLevel] = ...): ... + def __init__( + self, + report_level: Optional[ReportLevel] = ..., + ignore_directives: Optional[List[str]] = ..., + ignore_roles: Optional[List[str]] = ..., + ignore_substitutions: Optional[List[str]] = ..., + ignore_languages: Optional[List[str]] = ..., + ): ...