diff --git a/mathics/builtin/numeric.py b/mathics/builtin/numeric.py
index 63bf5d88e..73d625284 100644
--- a/mathics/builtin/numeric.py
+++ b/mathics/builtin/numeric.py
@@ -625,8 +625,9 @@ class RealValuedNumberQ(Builtin):
# No docstring since this is internal and it will mess up documentation.
# FIXME: Perhaps in future we will have a more explicite way to indicate not
# to add something to the docs.
+ no_doc = True
context = "Internal`"
-
+ summary_text = "test whether an expression is a real number"
rules = {
"Internal`RealValuedNumberQ[x_Real]": "True",
"Internal`RealValuedNumberQ[x_Integer]": "True",
@@ -639,6 +640,7 @@ class RealValuedNumericQ(Builtin):
# No docstring since this is internal and it will mess up documentation.
# FIXME: Perhaps in future we will have a more explicite way to indicate not
# to add something to the docs.
+ no_doc = True
context = "Internal`"
rules = {
diff --git a/mathics/doc/__init__.py b/mathics/doc/__init__.py
index 26efa89b8..1be93c620 100644
--- a/mathics/doc/__init__.py
+++ b/mathics/doc/__init__.py
@@ -1,8 +1,29 @@
# -*- coding: utf-8 -*-
"""
-Module for handling Mathics-style documentation.
+A module and library that assists in organizing document data
+located in static files and docstrings from
+Mathics3 Builtin Modules. Builtin Modules are written in Python and
+reside either in the Mathics3 core (mathics.builtin) or are packaged outside,
+in Mathics3 Modules e.g. pymathics.natlang.
-Right now this covers common LaTeX/PDF and routines common to
-Mathics Django. When this code is moved out, perhaps it will
-include the Mathics Django-specific piece.
+This data is stored in a way that facilitates:
+* organizing information to produce a LaTeX file
+* running documentation tests
+* producing HTML-based documentation
+
+The command-line utility ``docpipeline.py``, loads the data from
+Python modules and static files, accesses the functions here.
+
+Mathics Django also uses this library for its HTML-based documentation.
+
+The Mathics3 builtin function ``Information[]`` also uses to provide the
+information it reports.
+As with reading in data, final assembly to a LaTeX file or running
+documentation tests is done elsewhere.
+
+FIXME: This code should be replaced by Sphinx and autodoc.
+Things are such a mess, that it is too difficult to contemplate this right now.
+Also there higher-priority flaws that are more more pressing.
+In the shorter, we might we move code for extracting printing to a
+separate package.
"""
diff --git a/mathics/doc/common_doc.py b/mathics/doc/common_doc.py
index 0a3e041af..189f5347f 100644
--- a/mathics/doc/common_doc.py
+++ b/mathics/doc/common_doc.py
@@ -1,1040 +1,60 @@
# -*- coding: utf-8 -*-
"""
-A module and library that assists in organizing document data
-located in static files and docstrings from
-Mathics3 Builtin Modules. Builtin Modules are written in Python and
-reside either in the Mathics3 core (mathics.builtin) or are packaged outside,
-in Mathics3 Modules e.g. pymathics.natlang.
-This data is stored in a way that facilitates:
-* organizing information to produce a LaTeX file
-* running documentation tests
-* producing HTML-based documentation
+common_doc
-The command-line utility ``docpipeline.py``, loads the data from
-Python modules and static files, accesses the functions here.
+This module is kept for backward compatibility.
-Mathics Django also uses this library for its HTML-based documentation.
+The module was splitted into
+* mathics.doc.doc_entries: classes contaning the documentation entries and doctests.
+* mathics.doc.structure: the classes describing the elements in the documentation organization
+* mathics.doc.gather: functions to gather information from modules to build the
+ documentation reference.
-The Mathics3 builtin function ``Information[]`` also uses to provide the
-information it reports.
-As with reading in data, final assembly to a LaTeX file or running
-documentation tests is done elsewhere.
-
-FIXME: This code should be replaced by Sphinx and autodoc.
-Things are such a mess, that it is too difficult to contemplate this right now.
-Also there higher-priority flaws that are more more pressing.
-In the shorter, we might we move code for extracting printing to a
-separate package.
"""
-import logging
-import os.path as osp
-import pkgutil
-import re
-from os import environ, listdir
-from types import ModuleType
-from typing import Iterator, List, Optional, Tuple
-from mathics import settings
-from mathics.core.builtin import check_requires_list
-from mathics.core.load_builtin import (
- builtins_by_module as global_builtins_by_module,
- mathics3_builtins_modules,
-)
-from mathics.core.util import IS_PYPY
from mathics.doc.doc_entries import (
+ ALLOWED_TAGS,
+ ALLOWED_TAGS_RE,
+ CONSOLE_RE,
+ DL_ITEM_RE,
+ DL_RE,
+ HYPERTEXT_RE,
+ IMG_PNG_RE,
+ IMG_RE,
+ LATEX_RE,
+ LIST_ITEM_RE,
+ LIST_RE,
+ MATHICS_RE,
+ PYTHON_RE,
+ QUOTATIONS_RE,
+ REF_RE,
+ SPECIAL_COMMANDS,
+ DocTest,
+ DocTests,
+ DocText,
DocumentationEntry,
Tests,
- filter_comments,
+ get_results_by_test,
parse_docstring_to_DocumentationEntry_items,
+ post_sub,
+ pre_sub,
)
-from mathics.doc.utils import slugify
-from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules
-
-CHAPTER_RE = re.compile('(?s)(.*?)')
-SECTION_RE = re.compile('(?s)(.*?)')
-SUBSECTION_RE = re.compile('(?s)')
-
-# Debug flags.
-
-# Set to True if want to follow the process
-# The first phase is building the documentation data structure
-# based on docstrings:
-
-MATHICS_DEBUG_DOC_BUILD: bool = "MATHICS_DEBUG_DOC_BUILD" in environ
-
-# After building the doc structure, we extract test cases.
-MATHICS_DEBUG_TEST_CREATE: bool = "MATHICS_DEBUG_TEST_CREATE" in environ
-
-# Name of the Mathics3 Module part of the document.
-MATHICS3_MODULES_TITLE = "Mathics3 Modules"
-
-
-def get_module_doc(module: ModuleType) -> Tuple[str, str]:
- """
- Determine the title and text associated to the documentation
- of a module.
- If the module has a module docstring, extract the information
- from it. If not, pick the title from the name of the module.
- """
- doc = module.__doc__
- if doc is not None:
- doc = doc.strip()
- if doc:
- title = doc.splitlines()[0]
- text = "\n".join(doc.splitlines()[1:])
- else:
- title = module.__name__
- for prefix in ("mathics.builtin.", "mathics.optional.", "pymathics."):
- if title.startswith(prefix):
- title = title[len(prefix) :]
- title = title.capitalize()
- text = ""
- return title, text
-
-
-def get_submodule_names(obj) -> list:
- """Many builtins are organized into modules which, from a documentation
- standpoint, are like Mathematica Online Guide Docs.
-
- "List Functions", "Colors", or "Distance and Similarity Measures"
- are some examples Guide Documents group group various Builtin Functions,
- under submodules relate to that general classification.
-
- Here, we want to return a list of the Python modules under a "Guide Doc"
- module.
-
- As an example of a "Guide Doc" and its submodules, consider the
- module named mathics.builtin.colors. It collects code and documentation pertaining
- to the builtin functions that would be found in the Guide documentation for "Colors".
-
- The `mathics.builtin.colors` module has a submodule
- `mathics.builtin.colors.named_colors`.
-
- The builtin functions defined in `named_colors` then are those found in the
- "Named Colors" group of the "Colors" Guide Doc.
-
- So in this example then, in the list the modules returned for
- Python module `mathics.builtin.colors` would be the
- `mathics.builtin.colors.named_colors` module which contains the
- definition and docs for the "Named Colors" Mathics Bultin
- Functions.
- """
- modpkgs = []
- if hasattr(obj, "__path__"):
- for importer, modname, ispkg in pkgutil.iter_modules(obj.__path__):
- modpkgs.append(modname)
- modpkgs.sort()
- return modpkgs
-
-
-def get_doc_name_from_module(module) -> str:
- """
- Get the title associated to the module.
- If the module has a docstring, pick the name from
- its first line (the title). Otherwise, use the
- name of the module.
- """
- name = "???"
- if module.__doc__:
- lines = module.__doc__.strip()
- if not lines:
- name = module.__name__
- else:
- name = lines.split("\n")[0]
- return name
-
-
-def skip_doc(cls) -> bool:
- """Returns True if we should skip cls in docstring extraction."""
- return cls.__name__.endswith("Box") or (hasattr(cls, "no_doc") and cls.no_doc)
-
-
-def skip_module_doc(module, must_be_skipped) -> bool:
- """True if the module should not be included in the documentation"""
- return (
- module.__doc__ is None
- or module in must_be_skipped
- or module.__name__.split(".")[0] not in ("mathics", "pymathics")
- or hasattr(module, "no_doc")
- and module.no_doc
- )
-
-
-# DocSection has to appear before DocGuideSection which uses it.
-class DocSection:
- """An object for a Documented Section.
- A Section is part of a Chapter. It can contain subsections.
- """
-
- def __init__(
- self,
- chapter,
- title: str,
- text: str,
- operator,
- installed: bool = True,
- in_guide: bool = False,
- summary_text: str = "",
- ):
- self.chapter = chapter
- self.in_guide = in_guide
- self.installed = installed
- self.items = [] # tests in section when this is under a guide section
- self.operator = operator
- self.slug = slugify(title)
- self.subsections = []
- self.subsections_by_slug = {}
- self.summary_text = summary_text
- self.tests = None # tests in section when not under a guide section
- self.title = title
-
- if text.count("") != text.count("
"):
- raise ValueError(
- "Missing opening or closing tag in "
- "{} documentation".format(title)
- )
-
- # Needs to come after self.chapter is initialized since
- # DocumentationEntry uses self.chapter.
- # Notice that we need the documentation object, to have access
- # to the suitable subclass of DocumentationElement.
- documentation = self.chapter.part.doc
- self.doc = documentation.doc_class(text, title, None).set_parent_path(self)
-
- chapter.sections_by_slug[self.slug] = self
- if MATHICS_DEBUG_DOC_BUILD:
- print(" DEBUG Creating Section", title)
-
- # Add __eq__ and __lt__ so we can sort Sections.
- def __eq__(self, other) -> bool:
- return self.title == other.title
-
- def __lt__(self, other) -> bool:
- return self.title < other.title
-
- def __str__(self) -> str:
- return f" == {self.title} ==\n{self.doc}"
-
- def get_tests(self):
- """yield tests"""
- if self.installed:
- for test in self.doc.get_tests():
- yield test
-
- @property
- def parent(self):
- return self.chapter
-
- @parent.setter
- def parent(self, value):
- raise TypeError("parent is a read-only property")
-
-
-# DocChapter has to appear before DocGuideSection which uses it.
-class DocChapter:
- """An object for a Documented Chapter.
- A Chapter is part of a Part[dChapter. It can contain (Guide or plain) Sections.
- """
-
- def __init__(self, part, title, doc=None, chapter_order: Optional[int] = None):
- self.chapter_order = chapter_order
- self.doc = doc
- self.guide_sections = []
- self.part = part
- self.title = title
- self.slug = slugify(title)
- self.sections = []
- self.sections_by_slug = {}
- self.sort_order = None
- if doc:
- self.doc.set_parent_path(self)
-
- part.chapters_by_slug[self.slug] = self
-
- if MATHICS_DEBUG_DOC_BUILD:
- print(" DEBUG Creating Chapter", title)
-
- def __str__(self) -> str:
- """
- A DocChapter is represented as the index of its sections
- and subsections.
- """
- sections_descr = ""
- for section in self.all_sections:
- sec_class = "@>" if isinstance(section, DocGuideSection) else "@ "
- sections_descr += f" {sec_class} " + section.title + "\n"
- for subsection in section.subsections:
- sections_descr += " * " + subsection.title + "\n"
-
- return f" = {self.part.title}: {self.title} =\n\n{sections_descr}"
-
- @property
- def all_sections(self):
- return sorted(self.sections + self.guide_sections)
-
- @property
- def parent(self):
- return self.part
-
- @parent.setter
- def parent(self, value):
- raise TypeError("parent is a read-only property")
-
-
-class DocGuideSection(DocSection):
- """An object for a Documented Guide Section.
- A Guide Section is part of a Chapter. "Colors" or "Special Functions"
- are examples of Guide Sections, and each contains a number of Sections.
- like NamedColors or Orthogonal Polynomials.
- """
-
- def __init__(
- self,
- chapter: DocChapter,
- title: str,
- text: str,
- submodule,
- installed: bool = True,
- ):
- super().__init__(chapter, title, text, None, installed, False)
- self.section = submodule
-
- if MATHICS_DEBUG_DOC_BUILD:
- print(" DEBUG Creating Guide Section", title)
-
- # FIXME: turn into a @property tests?
- def get_tests(self):
- """
- Tests included in a Guide.
- """
- # FIXME: The below is a little weird for Guide Sections.
- # Figure out how to make this clearer.
- # A guide section's subsection are Sections without the Guide.
- # it is *their* subsections where we generally find tests.
- #
- # Currently, this is not called in docpipeline or in making
- # the LaTeX documentation.
- for section in self.subsections:
- if not section.installed:
- continue
- for subsection in section.subsections:
- # FIXME we are omitting the section title here...
- if not subsection.installed:
- continue
- for doctests in subsection.items:
- yield doctests.get_tests()
-
-
-def sorted_chapters(chapters: List[DocChapter]) -> List[DocChapter]:
- """Return chapters sorted by title"""
- return sorted(
- chapters,
- key=lambda chapter: str(chapter.sort_order)
- if chapter.sort_order is not None
- else chapter.title,
- )
-
-
-def sorted_modules(modules) -> list:
- """Return modules sorted by the ``sort_order`` attribute if that
- exists, or the module's name if not."""
- return sorted(
- modules,
- key=lambda module: module.sort_order
- if hasattr(module, "sort_order")
- else module.__name__,
- )
-
-
-class DocPart:
- """
- Represents one of the main parts of the document. Parts
- can be loaded from a mdoc file, generated automatically from
- the docstrings of Builtin objects under `mathics.builtin`.
- """
-
- chapter_class = DocChapter
-
- def __init__(self, doc, title, is_reference=False):
- self.doc = doc
- self.title = title
- self.chapters = []
- self.chapters_by_slug = {}
- self.is_reference = is_reference
- self.is_appendix = False
- self.slug = slugify(title)
- doc.parts_by_slug[self.slug] = self
- if MATHICS_DEBUG_DOC_BUILD:
- print("DEBUG Creating Part", title)
-
- def __str__(self) -> str:
- return f" Part {self.title}\n\n" + "\n\n".join(
- str(chapter) for chapter in sorted_chapters(self.chapters)
- )
-
-
-class Documentation:
- """
- `Documentation` describes an object containing the whole documentation system.
- Documentation
- |
- +--------0> Parts
- |
- +-----0> Chapters
- |
- +-----0>Sections
- | |
- | +------0> SubSections
- |
- +---->0>GuideSections
- |
- +-----0>Sections
- |
- +------0> SubSections
-
- (with 0>) meaning "aggregation".
-
- Each element contains a title, a collection of elements of the following class
- in the hierarchy. Parts, Chapters, Guide Sections, Sections and SubSections contains a doc
- attribute describing the content to be shown after the title, and before
- the elements of the subsequent terms in the hierarchy.
- """
-
- def __init__(self, title: str = "Title", doc_dir: str = ""):
- """
- Parameters
- ----------
- title : str, optional
- The title of the Documentation. The default is "Title".
- doc_dir : str, optional
- The path where the sources can be loaded. The default is "",
- meaning that no sources must be loaded.
- """
- # This is a way to load the default classes
- # without defining these attributes as class
- # attributes.
- self._set_classes()
- self.appendix = []
- self.doc_dir = doc_dir
- self.parts = []
- self.parts_by_slug = {}
- self.title = title
-
- def _set_classes(self):
- """
- Set the classes of the subelements. Must be overloaded
- by the subclasses.
- """
- if not hasattr(self, "part_class"):
- self.chapter_class = DocChapter
- self.doc_class = DocumentationEntry
- self.guide_section_class = DocGuideSection
- self.part_class = DocPart
- self.section_class = DocSection
- self.subsection_class = DocSubsection
-
- def __str__(self):
- result = self.title + "\n" + len(self.title) * "~" + "\n"
- return (
- result + "\n\n".join([str(part) for part in self.parts]) + "\n" + 60 * "-"
- )
-
- def add_section(
- self,
- chapter,
- section_name: str,
- section_object,
- operator,
- is_guide: bool = False,
- in_guide: bool = False,
- summary_text="",
- ):
- """
- Adds a DocSection or DocGuideSection
- object to the chapter, a DocChapter object.
- "section_object" is either a Python module or a Class object instance.
- """
- if section_object is not None:
- required_libs = getattr(section_object, "requires", [])
- installed = check_requires_list(required_libs) if required_libs else True
- # FIXME add an additional mechanism in the module
- # to allow a docstring and indicate it is not to go in the
- # user manual
- if not section_object.__doc__:
- return
-
- else:
- installed = True
-
- if is_guide:
- section = self.guide_section_class(
- chapter,
- section_name,
- section_object.__doc__,
- section_object,
- installed=installed,
- )
- chapter.guide_sections.append(section)
- else:
- section = self.section_class(
- chapter,
- section_name,
- section_object.__doc__,
- operator=operator,
- installed=installed,
- in_guide=in_guide,
- summary_text=summary_text,
- )
- chapter.sections.append(section)
-
- return section
-
- def add_subsection(
- self,
- chapter,
- section,
- subsection_name: str,
- instance,
- operator=None,
- in_guide=False,
- ):
- """
- Append a subsection for ``instance`` into ``section.subsections``
- """
-
- required_libs = getattr(instance, "requires", [])
- installed = check_requires_list(required_libs) if required_libs else True
-
- # FIXME add an additional mechanism in the module
- # to allow a docstring and indicate it is not to go in the
- # user manual
- if not instance.__doc__:
- return
- summary_text = (
- instance.summary_text if hasattr(instance, "summary_text") else ""
- )
- subsection = self.subsection_class(
- chapter,
- section,
- subsection_name,
- instance.__doc__,
- operator=operator,
- installed=installed,
- in_guide=in_guide,
- summary_text=summary_text,
- )
- section.subsections.append(subsection)
-
- def doc_part(self, title, modules, builtins_by_module, start):
- """
- Build documentation structure for a "Part" - Reference
- section or collection of Mathics3 Modules.
- """
-
- builtin_part = self.part_class(self, title, is_reference=start)
-
- # This is used to ensure that we pass just once over each module.
- # The algorithm we use to walk all the modules without repetitions
- # relies on this, which in my opinion is hard to test and susceptible
- # to errors. I guess we include it as a temporal fixing to handle
- # packages inside ``mathics.builtin``.
- modules_seen = set([])
-
- def filter_toplevel_modules(module_list):
- """
- Keep just the modules at the top level.
- """
- if len(module_list) == 0:
- return module_list
-
- modules_and_levels = sorted(
- ((module.__name__.count("."), module) for module in module_list),
- key=lambda x: x[0],
- )
- top_level = modules_and_levels[0][0]
- return (entry[1] for entry in modules_and_levels if entry[0] == top_level)
-
- # The loop to load chapters must be run over the top-level modules. Otherwise,
- # modules like ``mathics.builtin.functional.apply_fns_to_lists`` are loaded
- # as chapters and sections of a GuideSection, producing duplicated tests.
- #
- # Also, this provides a more deterministic way to walk the module hierarchy,
- # which can be decomposed in the way proposed in #984.
-
- modules = filter_toplevel_modules(modules)
- for module in sorted_modules(modules):
- if skip_module_doc(module, modules_seen):
- continue
- chapter = self.doc_chapter(module, builtin_part, builtins_by_module)
- if chapter is None:
- continue
- builtin_part.chapters.append(chapter)
-
- self.parts.append(builtin_part)
-
- def doc_chapter(self, module, part, builtins_by_module) -> Optional[DocChapter]:
- """
- Build documentation structure for a "Chapter" - reference section which
- might be a Mathics Module.
- """
- modules_seen = set([])
-
- title, text = get_module_doc(module)
- chapter = self.chapter_class(part, title, self.doc_class(text, title, None))
- builtins = builtins_by_module.get(module.__name__)
- if module.__file__.endswith("__init__.py"):
- # We have a Guide Section.
-
- # This is used to check if a symbol is not duplicated inside
- # a guide.
- submodule_names_seen = set([])
- name = get_doc_name_from_module(module)
- guide_section = self.add_section(
- chapter, name, module, operator=None, is_guide=True
- )
- submodules = [
- value
- for value in module.__dict__.values()
- if isinstance(value, ModuleType)
- ]
-
- # Add sections in the guide section...
- for submodule in sorted_modules(submodules):
- if skip_module_doc(submodule, modules_seen):
- continue
- elif IS_PYPY and submodule.__name__ == "builtins":
- # PyPy seems to add this module on its own,
- # but it is not something that can be importable
- continue
-
- submodule_name = get_doc_name_from_module(submodule)
- if submodule_name in submodule_names_seen:
- continue
- section = self.add_section(
- chapter,
- submodule_name,
- submodule,
- operator=None,
- is_guide=False,
- in_guide=True,
- )
- modules_seen.add(submodule)
- submodule_names_seen.add(submodule_name)
- guide_section.subsections.append(section)
-
- builtins = builtins_by_module.get(submodule.__name__, [])
- subsections = list(builtins)
- for instance in subsections:
- if hasattr(instance, "no_doc") and instance.no_doc:
- continue
-
- name = instance.get_name(short=True)
- if name in submodule_names_seen:
- continue
-
- submodule_names_seen.add(name)
- modules_seen.add(instance)
-
- self.add_subsection(
- chapter,
- section,
- name,
- instance,
- instance.get_operator(),
- in_guide=True,
- )
- else:
- if not builtins:
- return None
- sections = [
- builtin for builtin in builtins if not skip_doc(builtin.__class__)
- ]
- self.doc_sections(sections, modules_seen, chapter)
- return chapter
-
- def doc_sections(self, sections, modules_seen, chapter):
- """
- Load sections from a list of mathics builtins.
- """
-
- for instance in sections:
- if instance not in modules_seen and (
- not hasattr(instance, "no_doc") or not instance.no_doc
- ):
- name = instance.get_name(short=True)
- summary_text = (
- instance.summary_text if hasattr(instance, "summary_text") else ""
- )
- self.add_section(
- chapter,
- name,
- instance,
- instance.get_operator(),
- is_guide=False,
- in_guide=False,
- summary_text=summary_text,
- )
- modules_seen.add(instance)
-
- def get_part(self, part_slug):
- """return a section from part key"""
- return self.parts_by_slug.get(part_slug)
-
- def get_chapter(self, part_slug, chapter_slug):
- """return a section from part and chapter keys"""
- part = self.parts_by_slug.get(part_slug)
- if part:
- return part.chapters_by_slug.get(chapter_slug)
- return None
-
- def get_section(self, part_slug, chapter_slug, section_slug):
- """return a section from part, chapter and section keys"""
- part = self.parts_by_slug.get(part_slug)
- if part:
- chapter = part.chapters_by_slug.get(chapter_slug)
- if chapter:
- return chapter.sections_by_slug.get(section_slug)
- return None
-
- def get_subsection(self, part_slug, chapter_slug, section_slug, subsection_slug):
- """
- return a section from part, chapter, section and subsection
- keys
- """
- part = self.parts_by_slug.get(part_slug)
- if part:
- chapter = part.chapters_by_slug.get(chapter_slug)
- if chapter:
- section = chapter.sections_by_slug.get(section_slug)
- if section:
- return section.subsections_by_slug.get(subsection_slug)
-
- return None
-
- # FIXME: turn into a @property tests?
- def get_tests(self) -> Iterator:
- """
- Returns a generator to extracts lists test objects.
- """
- for part in self.parts:
- for chapter in sorted_chapters(part.chapters):
- if MATHICS_DEBUG_TEST_CREATE:
- print(f"DEBUG Gathering tests for Chapter {chapter.title}")
-
- tests = chapter.doc.get_tests()
- if tests:
- yield Tests(part.title, chapter.title, "", tests)
-
- for section in chapter.all_sections:
- if section.installed:
- if MATHICS_DEBUG_TEST_CREATE:
- if isinstance(section, DocGuideSection):
- print(
- f"DEBUG Gathering tests for Guide Section {section.title}"
- )
- else:
- print(
- f"DEBUG Gathering tests for Section {section.title}"
- )
-
- if isinstance(section, DocGuideSection):
- for docsection in section.subsections:
- for docsubsection in docsection.subsections:
- # FIXME: Something is weird here where tests for subsection items
- # appear not as a collection but individually and need to be
- # iterated below. Probably some other code is faulty and
- # when fixed the below loop and collection into doctest_list[]
- # will be removed.
- if not docsubsection.installed:
- continue
- doctest_list = []
- index = 1
- for doctests in docsubsection.items:
- doctest_list += list(doctests.get_tests())
- for test in doctest_list:
- test.index = index
- index += 1
-
- if doctest_list:
- yield Tests(
- section.chapter.part.title,
- section.chapter.title,
- docsubsection.title,
- doctest_list,
- )
- else:
- tests = section.doc.get_tests()
- if tests:
- yield Tests(
- part.title,
- chapter.title,
- section.title,
- tests,
- )
- return
-
- def load_documentation_sources(self):
- """
- Extract doctest data from various static XML-like doc files, Mathics3 Built-in functions
- (inside mathics.builtin), and external Mathics3 Modules.
-
- The extracted structure is stored in ``self``.
- """
- assert (
- len(self.parts) == 0
- ), "The documentation must be empty to call this function."
-
- # First gather data from static XML-like files. This constitutes "Part 1" of the
- # documentation.
- files = listdir(self.doc_dir)
- files.sort()
-
- chapter_order = 0
- for file in files:
- part_title = file[2:]
- if part_title.endswith(".mdoc"):
- part_title = part_title[: -len(".mdoc")]
- # If the filename start with a number, then is a main part. Otherwise
- # is an appendix.
- is_appendix = not file[0].isdigit()
- chapter_order = self.load_part_from_file(
- osp.join(self.doc_dir, file),
- part_title,
- chapter_order,
- is_appendix,
- )
-
- # Next extract data that has been loaded into Mathics3 when it runs.
- # This is information from `mathics.builtin`.
- # This is Part 2 of the documentation.
-
- # Notice that in order to generate the documentation
- # from the builtin classes, it is needed to call first to
- # import_and_load_builtins()
-
- for title, modules, builtins_by_module, start in [
- (
- "Reference of Built-in Symbols",
- mathics3_builtins_modules,
- global_builtins_by_module,
- True,
- )
- ]:
- self.doc_part(title, modules, builtins_by_module, start)
-
- # Next extract external Mathics3 Modules that have been loaded via
- # LoadModule, or eval_LoadModule. This is Part 3 of the documentation.
-
- self.doc_part(
- MATHICS3_MODULES_TITLE,
- pymathics_modules,
- pymathics_builtins_by_module,
- True,
- )
-
- # Finally, extract Appendix information. This include License text
- # This is the final Part of the documentation.
-
- for part in self.appendix:
- self.parts.append(part)
-
- return
-
- def load_part_from_file(
- self,
- filename: str,
- title: str,
- chapter_order: int,
- is_appendix: bool = False,
- ) -> int:
- """Load a markdown file as a part of the documentation"""
- part = self.part_class(self, title)
- text = open(filename, "rb").read().decode("utf8")
- text = filter_comments(text)
- chapters = CHAPTER_RE.findall(text)
- for title, text in chapters:
- chapter = self.chapter_class(part, title, chapter_order=chapter_order)
- chapter_order += 1
- text += ''
- section_texts = SECTION_RE.findall(text)
- for pre_text, title, text in section_texts:
- if title:
- section = self.section_class(
- chapter, title, text, operator=None, installed=True
- )
- chapter.sections.append(section)
- subsections = SUBSECTION_RE.findall(text)
- for subsection_title in subsections:
- subsection = self.subsection_class(
- chapter,
- section,
- subsection_title,
- text,
- )
- section.subsections.append(subsection)
- else:
- section = None
- if not chapter.doc:
- chapter.doc = self.doc_class(pre_text, title, section)
- pass
-
- part.chapters.append(chapter)
- if is_appendix:
- part.is_appendix = True
- self.appendix.append(part)
- else:
- self.parts.append(part)
- return chapter_order
-
-
-class DocSubsection:
- """An object for a Documented Subsection.
- A Subsection is part of a Section.
- """
-
- def __init__(
- self,
- chapter,
- section,
- title,
- text,
- operator=None,
- installed=True,
- in_guide=False,
- summary_text="",
- ):
- """
- Information that goes into a subsection object. This can be a written text, or
- text extracted from the docstring of a builtin module or class.
-
- About some of the parameters...
-
- Some subsections are contained in a grouping module and need special work to
- get the grouping module name correct.
-
- For example the Chapter "Colors" is a module so the docstring text for it is in
- mathics/builtin/colors/__init__.py . In mathics/builtin/colors/named-colors.py we have
- the "section" name for the class Red (the subsection) inside it.
- """
- title_summary_text = re.split(" -- ", title)
- n = len(title_summary_text)
- # We need the documentation object, to have access
- # to the suitable subclass of DocumentationElement.
- documentation = chapter.part.doc
-
- self.title = title_summary_text[0] if n > 0 else ""
- self.summary_text = title_summary_text[1] if n > 1 else summary_text
- self.doc = documentation.doc_class(text, title, None)
- self.chapter = chapter
- self.in_guide = in_guide
- self.installed = installed
- self.operator = operator
-
- self.section = section
- self.slug = slugify(title)
- self.subsections = []
- self.title = title
- self.doc.set_parent_path(self)
-
- # This smells wrong: Here a DocSection (a level in the documentation system)
- # is mixed with a DocumentationEntry. `items` is an attribute of the
- # `DocumentationEntry`, not of a Part / Chapter/ Section.
- # The content of a subsection should be stored in self.doc,
- # and the tests should set the rute (key) through self.doc.set_parent_doc
- if in_guide:
- # Tests haven't been picked out yet from the doc string yet.
- # Gather them here.
- self.items = self.doc.items
-
- for item in self.items:
- for test in item.get_tests():
- assert test.key is not None
- else:
- self.items = []
-
- if text.count("") != text.count("
"):
- raise ValueError(
- "Missing opening or closing tag in "
- "{} documentation".format(title)
- )
- self.section.subsections_by_slug[self.slug] = self
-
- if MATHICS_DEBUG_DOC_BUILD:
- print(" DEBUG Creating Subsection", title)
-
- def __str__(self) -> str:
- return f"=== {self.title} ===\n{self.doc}"
-
- @property
- def parent(self):
- return self.section
-
- @parent.setter
- def parent(self, value):
- raise TypeError("parent is a read-only property")
-
- def get_tests(self):
- """yield tests"""
- if self.installed:
- for test in self.doc.get_tests():
- yield test
-
-
-class MathicsMainDocumentation(Documentation):
- """
- MathicsMainDocumentation specializes ``Documentation`` by providing the attributes
- and methods needed to generate the documentation from the Mathics library.
-
- The parts of the documentation are loaded from the Markdown files contained
- in the path specified by ``self.doc_dir``. Files with names starting in numbers
- are considered parts of the main text, while those that starts with other characters
- are considered as appendix parts.
-
- In addition to the parts loaded from markdown files, a ``Reference of Builtin-Symbols`` part
- and a part for the loaded Pymathics modules are automatically generated.
-
- In the ``Reference of Built-in Symbols`` tom-level modules and files in ``mathics.builtin``
- are associated to Chapters. For single file submodules (like ``mathics.builtin.procedure``)
- The chapter contains a Section for each Symbol in the module. For sub-packages
- (like ``mathics.builtin.arithmetic``) sections are given by the sub-module files,
- and the symbols in these sub-packages defines the Subsections. ``__init__.py`` in
- subpackages are associated to GuideSections.
-
- In a similar way, in the ``Pymathics`` part, each ``pymathics`` module defines a Chapter,
- files in the module defines Sections, and Symbols defines Subsections.
-
-
- ``MathicsMainDocumentation`` is also used for creating test data and saving it to a
- Python Pickle file and running tests that appear in the documentation (doctests).
-
- There are other classes DjangoMathicsDocumentation and LaTeXMathicsDocumentation
- that format the data accumulated here. In fact I think those can sort of serve
- instead of this.
-
- """
-
- def __init__(self):
- super().__init__(title="Mathics Main Documentation", doc_dir=settings.DOC_DIR)
- self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL
- self.pymathics_doc_loaded = False
- self.doc_data_file = settings.get_doctest_latex_data_path(
- should_be_readable=True
- )
-
- def gather_doctest_data(self):
- """
- Populates the documentatation.
- (deprecated)
- """
- logging.warning(
- "gather_doctest_data is deprecated. Use load_documentation_sources"
- )
- return self.load_documentation_sources()
-
-
-# Backward compatibility
gather_tests = parse_docstring_to_DocumentationEntry_items
XMLDOC = DocumentationEntry
+
+from mathics.doc.structure import (
+ MATHICS3_MODULES_TITLE,
+ SUBSECTION_END_RE,
+ SUBSECTION_RE,
+ DocChapter,
+ DocGuideSection,
+ DocPart,
+ DocSection,
+ DocSubsection,
+ Documentation,
+ MathicsMainDocumentation,
+ sorted_chapters,
+)
diff --git a/mathics/doc/doc_entries.py b/mathics/doc/doc_entries.py
index 1d423d1dd..cef8ab37e 100644
--- a/mathics/doc/doc_entries.py
+++ b/mathics/doc/doc_entries.py
@@ -69,6 +69,18 @@
LIST_RE = re.compile(r"(?s)<(?Pul|ol)>(?P.*?)")
MATHICS_RE = re.compile(r"(?.*?)\]\((?P.*?)\)")
+MD_IMG_LABEL_RE = re.compile(r"!\[(?P.*?)\]\((?P<src>.*?)\)\{\#(?P<label>.*?)\}")
+MD_PYTHON_RE = re.compile(
+ r"``\s*[pP]ython\n(?P<pythoncode>.*?)``", re.DOTALL | re.MULTILINE
+)
+MD_REF_RE = re.compile(r"\[(?P<label>.*?)\]\((?P<url>.*?)\)")
+MD_URL_RE = re.compile(r"\<(?P<prot>http|https|ftp|mail?)\:\/\/(?P<url>.*?)\>")
+
+MD_TAG_RE = re.compile(r"[{]\#(?P<label>.*?)[}]")
+
+
PYTHON_RE = re.compile(r"(?s)<python>(.*?)</python>")
QUOTATIONS_RE = re.compile(r"\"([\w\s,]*?)\"")
REF_RE = re.compile(r'<ref label="(?P<label>.*?)">')
@@ -84,7 +96,6 @@
"Wolfram": (r"<em>Wolfram</em>", r"\emph{Wolfram}"),
"skip": (r"<br /><br />", r"\bigskip"),
}
-SUBSECTION_END_RE = re.compile("</subsection>")
TESTCASE_RE = re.compile(
@@ -97,6 +108,68 @@
TESTCASE_OUT_RE = re.compile(r"^\s*([:|=])(.*)$")
+# TODO: Check if it wouldn't be better to go in the opposite direction,
+# to have a ReStructured markdown compliant syntax everywhere.
+def markdown_to_native(text):
+ """
+ This function converts common markdown syntax into
+ the Mathics XML native documentation syntax.
+ """
+ text, post_substitutions = pre_sub(
+ MD_PYTHON_RE, text, lambda m: "<python>%s</python>" % m.group(1)
+ )
+
+ def repl_figs_with_label(match):
+ caption = match.group(1)
+ src = match.group(2)
+ label = match.group(3)
+ return (
+ r"<imgpng src="
+ f"'{src}'"
+ " title="
+ f"'{caption}'"
+ " label="
+ f"'{label}'"
+ ">"
+ )
+
+ text = MD_IMG_LABEL_RE.sub(repl_figs_with_label, text)
+
+ def repl_figs(match):
+ caption = match.group(1)
+ src = match.group(2)
+ return r"<imgpng src=" f"'{src}'" " title=" f"'{caption}'" ">"
+
+ text = MD_IMG_RE.sub(repl_figs, text)
+
+ def repl_ref(match):
+ label = match.group(1)
+ reference = match.group(2)
+ return f"<url>:{label}:{reference}</url>"
+
+ text = MD_REF_RE.sub(repl_ref, text)
+
+ def repl_url(match):
+ prot = match.group(1)
+ reference = match.group(2)
+ return f"<url>{prot}://{reference}</url>"
+
+ text = MD_URL_RE.sub(repl_url, text)
+
+ def repl_labels(match):
+ label = match.group(1)
+ return r" \label{" f"{label}" "} "
+
+ text = MD_TAG_RE.sub(repl_labels, text)
+
+ def repl_python_code(match):
+ pass
+
+ text = MD_PYTHON_RE.sub(repl_python_code, text)
+
+ return post_sub(text, post_substitutions)
+
+
def get_results_by_test(test_expr: str, full_test_key: list, doc_data: dict) -> dict:
"""
Sometimes test numbering is off, either due to bugs or changes since the
@@ -137,7 +210,9 @@ def get_results_by_test(test_expr: str, full_test_key: list, doc_data: dict) ->
if result_candidate["query"] == test_expr:
if result:
# Already found something
- print(f"Warning, multiple results appear under {search_key}.")
+ logging.warning(
+ f"Warning, multiple results appear under {search_key}."
+ )
return {}
result = result_candidate
@@ -217,6 +292,10 @@ def parse_docstring_to_DocumentationEntry_items(
# Remove leading <dl>...</dl>
# doc = DL_RE.sub("", doc)
+ # Convert markdown syntax to XML native syntax.
+ # TODO: See if it wouldn't be better to go in the opposite way:
+ # convert the native syntax to a common-markdown compliant syntax.
+
# pre-substitute Python code because it might contain tests
doc, post_substitutions = pre_sub(
PYTHON_RE, doc, lambda m: "<python>%s</python>" % m.group(1)
@@ -393,7 +472,7 @@ class DocText:
"""
def __init__(self, text):
- self.text = text
+ self.text = markdown_to_native(text)
def __str__(self) -> str:
return self.text
diff --git a/mathics/doc/documentation/1-Manual.mdoc b/mathics/doc/documentation/1-Manual.mdoc
index d3357e2cd..cfd893ffd 100644
--- a/mathics/doc/documentation/1-Manual.mdoc
+++ b/mathics/doc/documentation/1-Manual.mdoc
@@ -1,12 +1,12 @@
<chapter title="Introduction">
-\Mathics---to be pronounced like "Mathematics" without the "emat"---is a general-purpose computer algebra system (CAS). It is meant to be a free, open-source alternative to \Mathematica. It is free both as in "free beer" and as in "freedom". Mathics can be run \Mathics locally, and to facilitate installation of the vast amount of software need to run this, there is a <url>:docker image available on dockerhub: https://hub.docker.com/r/mathicsorg/mathics</url>.
+\Mathics---to be pronounced like "Mathematics" without the "emat"---is a general-purpose computer algebra system (CAS). It is meant to be a free, open-source alternative to \Mathematica. It is free both as in "free beer" and as in "freedom". Mathics can be run \Mathics locally, and to facilitate installation of the vast amount of software need to run this, there is a [docker image available on dockerhub](https://hub.docker.com/r/mathicsorg/mathics).
The programming language of \Mathics is meant to resemble the \Wolfram Language as much as possible. However, \Mathics is in no way affiliated or supported by \Wolfram. \Mathics will probably never have the power to compete with \Mathematica in industrial applications; it is an alternative though. It also invites community development at all levels.
-See the <url>:installation instructions: https://mathics-development-guide.readthedocs.io/en/latest/installing/index.html</url> for the most recent instructions for installing from PyPI, or the source.
+See the [installation instructions](https://mathics-development-guide.readthedocs.io/en/latest/installing/index.html) for the most recent instructions for installing from PyPI, or the source.
-For implementation details see <url>https://mathics-development-guide.readthedocs.io/en/latest/</url>.
+For implementation details see <https://mathics-development-guide.readthedocs.io/en/latest/>.
<section title="Why yet another CAS, one based on Mathematica?">
\Mathematica is great, but it a couple of disadvantages.
@@ -24,10 +24,10 @@ However, even if you are willing to pay hundreds of dollars for the software, yo
\Mathics aims at combining the best of both worlds: the beauty of \Mathematica backed by a free, extensible Python core which includes a rich set of Python tools including:
<ul>
- <li><url>:mpmath: https://mpmath.org/</url> for floating-point arithmetic with arbitrary precision,
- <li><url>:numpy: https://numpy.org/numpy</url> for numeric computation,
- <li><url>:SymPy: https://sympy.org</url> for symbolic mathematics, and
- <li>optionally <url>:SciPy: https://www.scipy.org/</url> for Scientific calculations.
+ <li>[mpmath](https://mpmath.org/) for floating-point arithmetic with arbitrary precision,
+ <li>[numpy](https://numpy.org/numpy) for numeric computation,
+ <li>[SymPy](https://sympy.org) for symbolic mathematics, and
+ <li>optionally [SciPy](https://www.scipy.org/) for Scientific calculations.
</ul>
Performance of \Mathics is not, right now, practical in large-scale projects and calculations. However can be used as a tool for exploration and education.
@@ -47,9 +47,9 @@ Outside of the "core" \Mathics kernel (which has a only primitive command-line i
<ul>
<li>a Django-based web server
<li>a command-line interface using either prompt-toolkit, or GNU Readline
- <li>a <url>:Mathics3 module for Graphs:https://pypi.org/project/pymathics-graph/</url> (via <url>:NetworkX:https://networkx.org/</url>),
- <li>a <url>:Mathics3 module for NLP:https://pypi.org/project/pymathics-natlang/</url> (via <url>:nltk:https://www.nltk.org/</url>, <url>:spacy:https://spacy.io/</url>, and others)
- <li>a <url>:A docker container:https://hub.docker.com/r/mathicsorg/mathics</url> which bundles all of the above
+ <li>a [Mathics3 module for Graphs](https://pypi.org/project/pymathics-graph/) (via [NetworkX](https://networkx.org/)),
+ <li>a [Mathics3 module for NLP](https://pypi.org/project/pymathics-natlang/) (via [nltk](https://www.nltk.org/), [spacy](https://spacy.io/), and others)
+ <li>a [A docker container](https://hub.docker.com/r/mathicsorg/mathics) which bundles all of the above
</ul>
</section>
@@ -216,7 +216,7 @@ The relative uncertainty of '3.1416`3' is 10^-3. It is numerically equivalent, i
>> 3.1416`3 == 3.1413`4
= True
-We can get the precision of the number by using the \Mathics Built-in function <url>:'Precision': /doc/reference-of-built-in-symbols/atomic-elements-of-expressions/representation-of-numbers/precision/</url>:
+We can get the precision of the number by using the \Mathics Built-in function <url>:'Precision': /doc/reference-of-built-in-symbols/atomic-elements-of-expressions/precision</url>:
>> Precision[3.1413`4]
= 4.
diff --git a/mathics/doc/gather.py b/mathics/doc/gather.py
new file mode 100644
index 000000000..4951ca8b6
--- /dev/null
+++ b/mathics/doc/gather.py
@@ -0,0 +1,374 @@
+# -*- coding: utf-8 -*-
+"""
+Gather module information
+
+Functions used to build the reference sections from module information.
+
+"""
+
+import importlib
+import os.path as osp
+import pkgutil
+from os import listdir
+from types import ModuleType
+from typing import Tuple, Union
+
+from mathics.core.builtin import Builtin, check_requires_list
+from mathics.core.util import IS_PYPY
+from mathics.doc.doc_entries import DocumentationEntry
+from mathics.doc.structure import DocChapter, DocGuideSection, DocSection, DocSubsection
+
+
+def check_installed(src: Union[ModuleType, Builtin]) -> bool:
+ """Check if the required libraries"""
+ required_libs = getattr(src, "requires", [])
+ return check_requires_list(required_libs) if required_libs else True
+
+
+def filter_toplevel_modules(module_list):
+ """
+ Keep just the modules at the top level.
+ """
+ if len(module_list) == 0:
+ return module_list
+
+ modules_and_levels = sorted(
+ ((module.__name__.count("."), module) for module in module_list),
+ key=lambda x: x[0],
+ )
+ top_level = modules_and_levels[0][0]
+ return (entry[1] for entry in modules_and_levels if entry[0] == top_level)
+
+
+def gather_docs_from_files(documentation, path):
+ """
+ Load documentation from files in path
+ """
+ # First gather data from static XML-like files. This constitutes "Part 1" of the
+ # documentation.
+ files = listdir(path)
+ files.sort()
+
+ chapter_order = 0
+ for file in files:
+ part_title = file[2:]
+ if part_title.endswith(".mdoc"):
+ part_title = part_title[: -len(".mdoc")]
+ # If the filename start with a number, then is a main part. Otherwise
+ # is an appendix.
+ is_appendix = not file[0].isdigit()
+ chapter_order = documentation.load_part_from_file(
+ osp.join(path, file),
+ part_title,
+ chapter_order,
+ is_appendix,
+ )
+
+
+def gather_reference_part(documentation, title, modules, builtins_by_module):
+ """
+ Build a part from a title, a list of modules and information
+ of builtins by modules.
+ """
+ part_class = documentation.part_class
+ reference_part = part_class(documentation, title, True)
+ modules = filter_toplevel_modules(modules)
+ for module in sorted_modules(modules):
+ if skip_module_doc(module):
+ continue
+ chapter = doc_chapter(module, reference_part, builtins_by_module)
+ if chapter is None:
+ continue
+ # reference_part.chapters.append(chapter)
+ return reference_part
+
+
+def doc_chapter(module, part, builtins_by_module):
+ """
+ Build documentation structure for a "Chapter" - reference section which
+ might be a Mathics Module.
+ """
+ # TODO: reformulate me in a way that symbols are always translated to
+ # sections, and guide sections do not contain subsections.
+ documentation = part.documentation if part else None
+ chapter_class = documentation.chapter_class if documentation else DocChapter
+ doc_class = documentation.doc_class if documentation else DocumentationEntry
+ title, text = get_module_doc(module)
+ chapter = chapter_class(part, title, doc_class(text, title, None))
+ part.chapters.append(chapter)
+
+ assert len(chapter.sections) == 0
+
+ # visited = set(sec.title for sec in symbol_sections)
+ # If the module is a package, add the guides and symbols from the submodules
+ if module.__file__.endswith("__init__.py"):
+ guide_sections, symbol_sections = gather_guides_and_sections(
+ chapter, module, builtins_by_module
+ )
+ chapter.guide_sections.extend(guide_sections)
+
+ for sec in symbol_sections:
+ if sec.title in visited:
+ print(sec.title, "already visited. Skipped.")
+ else:
+ visited.add(sec.title)
+ chapter.sections.append(sec)
+ else:
+ symbol_sections = gather_sections(chapter, module, builtins_by_module)
+ chapter.sections.extend(symbol_sections)
+
+ return chapter
+
+
+def gather_sections(chapter, module, builtins_by_module, section_class=None) -> list:
+ """Build a list of DocSections from a "top-level" module"""
+ symbol_sections = []
+ if skip_module_doc(module):
+ return []
+
+ part = chapter.part if chapter else None
+ documentation = part.documentation if part else None
+ if section_class is None:
+ section_class = documentation.section_class if documentation else DocSection
+
+ # TODO: Check the reason why for the module
+ # `mathics.builtin.numbers.constants`
+ # `builtins_by_module` has two copies of `Khinchin`.
+ # By now, we avoid the repetition by
+ # converting the entries into `set`s.
+ #
+ visited = set()
+ for symbol_instance in builtins_by_module[module.__name__]:
+ if skip_doc(symbol_instance, module):
+ continue
+ default_contexts = ("System`", "Pymathics`")
+ title = symbol_instance.get_name(
+ short=(symbol_instance.context in default_contexts)
+ )
+ if title in visited:
+ continue
+ visited.add(title)
+ text = symbol_instance.__doc__
+ operator = symbol_instance.get_operator()
+ installed = check_installed(symbol_instance)
+ summary_text = symbol_instance.summary_text
+ section = section_class(
+ chapter,
+ title,
+ text,
+ operator,
+ installed,
+ summary_text=summary_text,
+ )
+ assert (
+ section not in symbol_sections
+ ), f"{section.title} already in {module.__name__}"
+ symbol_sections.append(section)
+
+ return symbol_sections
+
+
+def gather_subsections(chapter, section, module, builtins_by_module) -> list:
+ """Build a list of DocSubsections from a "top-level" module"""
+
+ part = chapter.part if chapter else None
+ documentation = part.documentation if part else None
+ section_class = documentation.subsection_class if documentation else DocSubsection
+
+ def section_function(
+ chapter,
+ title,
+ text,
+ operator=None,
+ installed=True,
+ in_guide=False,
+ summary_text="",
+ ):
+ return section_class(
+ chapter, section, title, text, operator, installed, in_guide, summary_text
+ )
+
+ return gather_sections(chapter, module, builtins_by_module, section_function)
+
+
+def gather_guides_and_sections(chapter, module, builtins_by_module):
+ """
+ Look at the submodules in module, and produce the guide sections
+ and sections.
+ """
+ guide_sections = []
+ symbol_sections = []
+ if skip_module_doc(module):
+ return guide_sections, symbol_sections
+
+ if not module.__file__.endswith("__init__.py"):
+ return guide_sections, symbol_sections
+
+ # Determine the class for sections and guide sections
+ part = chapter.part if chapter else None
+ documentation = part.documentation if part else None
+ guide_class = (
+ documentation.guide_section_class if documentation else DocGuideSection
+ )
+
+ # Loop over submodules
+ docpath = f"/doc/{chapter.part.slug}/{chapter.slug}/"
+
+ for sub_module in submodules(module):
+ if skip_module_doc(sub_module):
+ continue
+
+ title, text = get_module_doc(sub_module)
+ installed = check_installed(sub_module)
+
+ guide_section = guide_class(
+ chapter=chapter,
+ title=title,
+ text=text,
+ submodule=sub_module,
+ installed=installed,
+ )
+
+ submodule_symbol_sections = gather_subsections(
+ chapter, guide_section, sub_module, builtins_by_module
+ )
+
+ guide_section.subsections.extend(submodule_symbol_sections)
+ guide_sections.append(guide_section)
+
+ # TODO, handle recursively the submodules.
+ # Here there I see two options:
+ # if sub_module.__file__.endswith("__init__.py"):
+ # (deeper_guide_sections,
+ # deeper_symbol_sections) = gather_guides_and_sections(chapter,
+ # sub_module, builtins_by_module)
+ # symbol_sections.extend(deeper_symbol_sections)
+ # guide_sections.extend(deeper_guide_sections)
+ return guide_sections, []
+
+
+def get_module_doc(module: ModuleType) -> Tuple[str, str]:
+ """
+ Determine the title and text associated to the documentation
+ of a module.
+ If the module has a module docstring, extract the information
+ from it. If not, pick the title from the name of the module.
+ """
+ doc = module.__doc__
+ if doc is not None:
+ doc = doc.strip()
+ if doc:
+ title = doc.splitlines()[0]
+ text = "\n".join(doc.splitlines()[1:])
+ else:
+ title = module.__name__
+ for prefix in ("mathics.builtin.", "mathics.optional.", "pymathics."):
+ if title.startswith(prefix):
+ title = title[len(prefix) :]
+ title = title.capitalize()
+ text = ""
+ return title, text
+
+
+def get_submodule_names(obj) -> list:
+ """Many builtins are organized into modules which, from a documentation
+ standpoint, are like Mathematica Online Guide Docs.
+
+ "List Functions", "Colors", or "Distance and Similarity Measures"
+ are some examples Guide Documents group group various Builtin Functions,
+ under submodules relate to that general classification.
+
+ Here, we want to return a list of the Python modules under a "Guide Doc"
+ module.
+
+ As an example of a "Guide Doc" and its submodules, consider the
+ module named mathics.builtin.colors. It collects code and documentation pertaining
+ to the builtin functions that would be found in the Guide documentation for "Colors".
+
+ The `mathics.builtin.colors` module has a submodule
+ `mathics.builtin.colors.named_colors`.
+
+ The builtin functions defined in `named_colors` then are those found in the
+ "Named Colors" group of the "Colors" Guide Doc.
+
+ So in this example then, in the list the modules returned for
+ Python module `mathics.builtin.colors` would be the
+ `mathics.builtin.colors.named_colors` module which contains the
+ definition and docs for the "Named Colors" Mathics Bultin
+ Functions.
+ """
+ modpkgs = []
+ if hasattr(obj, "__path__"):
+ for _, modname, __ in pkgutil.iter_modules(obj.__path__):
+ modpkgs.append(modname)
+ modpkgs.sort()
+ return modpkgs
+
+
+def get_doc_name_from_module(module) -> str:
+ """
+ Get the title associated to the module.
+ If the module has a docstring, pick the name from
+ its first line (the title). Otherwise, use the
+ name of the module.
+ """
+ name = "???"
+ if module.__doc__:
+ lines = module.__doc__.strip()
+ if not lines:
+ name = module.__name__
+ else:
+ name = lines.split("\n")[0]
+ return name
+
+
+def skip_doc(instance, module="") -> bool:
+ """Returns True if we should skip the docstring extraction."""
+ if not isinstance(module, str):
+ module = module.__name__ if module else ""
+
+ if type(instance).__name__.endswith("Box"):
+ return True
+ if hasattr(instance, "no_doc") and instance.no_doc:
+ return True
+
+ # Just include the builtins defined in the module.
+ if module:
+ if module != instance.__class__.__module__:
+ return True
+ return False
+
+
+def skip_module_doc(module, must_be_skipped=frozenset()) -> bool:
+ """True if the module should not be included in the documentation"""
+ if IS_PYPY and module.__name__ == "builtins":
+ return True
+ return (
+ module.__doc__ is None
+ or module in must_be_skipped
+ or module.__name__.split(".")[0] not in ("mathics", "pymathics")
+ or hasattr(module, "no_doc")
+ and module.no_doc
+ )
+
+
+def sorted_modules(modules) -> list:
+ """Return modules sorted by the ``sort_order`` attribute if that
+ exists, or the module's name if not."""
+ return sorted(
+ modules,
+ key=lambda module: module.sort_order
+ if hasattr(module, "sort_order")
+ else module.__name__,
+ )
+
+
+def submodules(package):
+ """Generator of the submodules in a package"""
+ package_folder = package.__file__[: -len("__init__.py")]
+ for _, module_name, __ in pkgutil.iter_modules([package_folder]):
+ try:
+ module = importlib.import_module(package.__name__ + "." + module_name)
+ except Exception:
+ continue
+ yield module
diff --git a/mathics/doc/latex/Makefile b/mathics/doc/latex/Makefile
index 82552c70f..a4f5060e7 100644
--- a/mathics/doc/latex/Makefile
+++ b/mathics/doc/latex/Makefile
@@ -37,7 +37,7 @@ logo-heptatom.pdf logo-text-nodrop.pdf:
(cd .. && $(BASH) ./images.sh)
#: The build of the documentation which is derived from docstrings in the Python code and doctest data
-documentation.tex: $(DOCTEST_LATEX_DATA_PCL)
+documentation.tex: $(DOCTEST_LATEX_DATA_PCL) ../documentation/1-Manual.mdoc
$(PYTHON) ./doc2latex.py $(MATHICS3_MODULE_OPTION) && $(BASH) ./sed-hack.sh
#: Same as mathics.pdf
diff --git a/mathics/doc/latex_doc.py b/mathics/doc/latex_doc.py
index 34cb977d9..466d2206c 100644
--- a/mathics/doc/latex_doc.py
+++ b/mathics/doc/latex_doc.py
@@ -6,17 +6,6 @@
import re
from typing import Optional
-from mathics.doc.common_doc import (
- SUBSECTION_RE,
- DocChapter,
- DocGuideSection,
- DocPart,
- DocSection,
- DocSubsection,
- Documentation,
- MathicsMainDocumentation,
- sorted_chapters,
-)
from mathics.doc.doc_entries import (
CONSOLE_RE,
DL_ITEM_RE,
@@ -32,7 +21,6 @@
QUOTATIONS_RE,
REF_RE,
SPECIAL_COMMANDS,
- SUBSECTION_END_RE,
DocTest,
DocTests,
DocText,
@@ -41,6 +29,18 @@
post_sub,
pre_sub,
)
+from mathics.doc.structure import (
+ SUBSECTION_END_RE,
+ SUBSECTION_RE,
+ DocChapter,
+ DocGuideSection,
+ DocPart,
+ DocSection,
+ DocSubsection,
+ Documentation,
+ MathicsMainDocumentation,
+ sorted_chapters,
+)
# We keep track of the number of \begin{asy}'s we see so that
# we can assocation asymptote file numbers with where they are
@@ -255,6 +255,8 @@ def repl_hypertext(match) -> str:
# then is is a link to a section
# in this manual, so use "\ref" rather than "\href'.
if content.find("/doc/") == 0:
+ slug = "/".join(content.split("/")[2:]).rstrip("/")
+ return "%s \\ref{%s}" % (text, latex_label_safe(slug))
slug = "/".join(content.split("/")[2:]).rstrip("/")
return "%s of section~\\ref{%s}" % (text, latex_label_safe(slug))
else:
@@ -647,6 +649,16 @@ def latex(
("\n\n\\chapter{%(title)s}\n\\chapterstart\n\n%(intro)s")
% {"title": escape_latex(self.title), "intro": intro},
"\\chaptersections\n",
+ # ####################
+ "\n\n".join(
+ section.latex(doc_data, quiet)
+ # Here we should use self.all_sections, but for some reason
+ # guidesections are not properly loaded, duplicating
+ # the load of subsections.
+ for section in sorted(self.guide_sections)
+ if not filter_sections or section.title in filter_sections
+ ),
+ # ###################
"\n\n".join(
section.latex(doc_data, quiet)
# Here we should use self.all_sections, but for some reason
@@ -725,11 +737,11 @@ def latex(self, doc_data: dict, quiet=False) -> str:
sections = "\n\n".join(section.latex(doc_data) for section in self.subsections)
slug = f"{self.chapter.part.slug}/{self.chapter.slug}/{self.slug}"
section_string = (
- "\n\n\\section*{%s}{%s}\n" % (title, index)
+ "\n\n\\section{%s}{%s}\n" % (title, index)
+ "\n\\label{%s}" % latex_label_safe(slug)
+ "\n\\sectionstart\n\n"
+ f"{content}"
- + ("\\addcontentsline{toc}{section}{%s}" % title)
+ # + ("\\addcontentsline{toc}{section}{%s}" % title)
+ sections
+ "\\sectionend"
)
@@ -788,7 +800,7 @@ def latex(self, doc_data: dict, quiet=False) -> str:
guide_sections = [
(
"\n\n\\section{%(title)s}\n\\sectionstart\n\n%(intro)s"
- "\\addcontentsline{toc}{section}{%(title)s}"
+ # "\\addcontentsline{toc}{section}{%(title)s}"
)
% {"title": escape_latex(self.title), "intro": intro},
"\n\n".join(section.latex(doc_data) for section in self.subsections),
diff --git a/mathics/doc/structure.py b/mathics/doc/structure.py
new file mode 100644
index 000000000..bdeb9ba42
--- /dev/null
+++ b/mathics/doc/structure.py
@@ -0,0 +1,704 @@
+# -*- coding: utf-8 -*-
+"""
+Structural elements of Mathics Documentation
+
+This module contains the classes representing the Mathics documentation structure.
+
+"""
+import logging
+import re
+from os import environ
+from typing import Iterator, List, Optional
+
+from mathics import settings
+from mathics.core.builtin import check_requires_list
+from mathics.core.load_builtin import (
+ builtins_by_module as global_builtins_by_module,
+ mathics3_builtins_modules,
+)
+from mathics.doc.doc_entries import DocumentationEntry, Tests, filter_comments
+from mathics.doc.utils import slugify
+from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules
+
+CHAPTER_RE = re.compile('(?s)<chapter title="(.*?)">(.*?)</chapter>')
+SECTION_RE = re.compile('(?s)(.*?)<section title="(.*?)">(.*?)</section>')
+SUBSECTION_RE = re.compile('(?s)<subsection title="(.*?)">')
+SUBSECTION_END_RE = re.compile("</subsection>")
+
+# Debug flags.
+
+# Set to True if want to follow the process
+# The first phase is building the documentation data structure
+# based on docstrings:
+
+MATHICS_DEBUG_DOC_BUILD: bool = "MATHICS_DEBUG_DOC_BUILD" in environ
+
+# After building the doc structure, we extract test cases.
+MATHICS_DEBUG_TEST_CREATE: bool = "MATHICS_DEBUG_TEST_CREATE" in environ
+
+# Name of the Mathics3 Module part of the document.
+MATHICS3_MODULES_TITLE = "Mathics3 Modules"
+
+
+# DocSection has to appear before DocGuideSection which uses it.
+class DocSection:
+ """An object for a Documented Section.
+ A Section is part of a Chapter. It can contain subsections.
+ """
+
+ def __init__(
+ self,
+ chapter,
+ title: str,
+ text: str,
+ operator,
+ installed: bool = True,
+ in_guide: bool = False,
+ summary_text: str = "",
+ ):
+ self.chapter = chapter
+ self.in_guide = in_guide
+ self.installed = installed
+ self.items = [] # tests in section when this is under a guide section
+ self.operator = operator
+ self.slug = slugify(title)
+ self.subsections = []
+ self.subsections_by_slug = {}
+ self.summary_text = summary_text
+ self.tests = None # tests in section when not under a guide section
+ self.title = title
+
+ if text.count("<dl>") != text.count("</dl>"):
+ raise ValueError(
+ "Missing opening or closing <dl> tag in "
+ "{} documentation".format(title)
+ )
+
+ # Needs to come after self.chapter is initialized since
+ # DocumentationEntry uses self.chapter.
+ # Notice that we need the documentation object, to have access
+ # to the suitable subclass of DocumentationElement.
+ documentation = self.chapter.part.documentation
+ self.doc = documentation.doc_class(text, title, None).set_parent_path(self)
+
+ chapter.sections_by_slug[self.slug] = self
+ if MATHICS_DEBUG_DOC_BUILD:
+ print(" DEBUG Creating Section", title)
+
+ # Add __eq__ and __lt__ so we can sort Sections.
+ def __eq__(self, other) -> bool:
+ return self.title == other.title
+
+ def __lt__(self, other) -> bool:
+ return self.title < other.title
+
+ def __str__(self) -> str:
+ return f" == {self.title} ==\n{self.doc}"
+
+ @property
+ def parent(self):
+ "the container where the section is"
+ return self.chapter
+
+ @parent.setter
+ def parent(self, value):
+ "the container where the section is"
+ raise TypeError("parent is a read-only property")
+
+ def get_tests(self):
+ """yield tests"""
+ if self.installed:
+ for test in self.doc.get_tests():
+ yield test
+
+
+# DocChapter has to appear before DocGuideSection which uses it.
+class DocChapter:
+ """An object for a Documented Chapter.
+ A Chapter is part of a Part[dChapter. It can contain (Guide or plain) Sections.
+ """
+
+ def __init__(self, part, title, doc=None, chapter_order: Optional[int] = None):
+ self.chapter_order = chapter_order
+ self.doc = doc
+ self.guide_sections = []
+ self.part = part
+ self.title = title
+ self.slug = slugify(title)
+ self.sections = []
+ self.sections_by_slug = {}
+ self.sort_order = None
+ if doc:
+ self.doc.set_parent_path(self)
+
+ part.chapters_by_slug[self.slug] = self
+
+ if MATHICS_DEBUG_DOC_BUILD:
+ print(" DEBUG Creating Chapter", title)
+
+ def __str__(self) -> str:
+ """
+ A DocChapter is represented as the index of its sections
+ and subsections.
+ """
+ sections_descr = ""
+ for section in self.all_sections:
+ sec_class = "@>" if isinstance(section, DocGuideSection) else "@ "
+ sections_descr += f" {sec_class} " + section.title + "\n"
+ for subsection in section.subsections:
+ sections_descr += " * " + subsection.title + "\n"
+
+ return f" = {self.part.title}: {self.title} =\n\n{sections_descr}"
+
+ @property
+ def all_sections(self):
+ "guides and normal sections"
+ return sorted(self.guide_sections) + sorted(self.sections)
+
+ @property
+ def parent(self):
+ "the container where the chapter is"
+ return self.part
+
+ @parent.setter
+ def parent(self, value):
+ "the container where the chapter is"
+ raise TypeError("parent is a read-only property")
+
+
+class DocGuideSection(DocSection):
+ """An object for a Documented Guide Section.
+ A Guide Section is part of a Chapter. "Colors" or "Special Functions"
+ are examples of Guide Sections, and each contains a number of Sections.
+ like NamedColors or Orthogonal Polynomials.
+ """
+
+ def __init__(
+ self,
+ chapter: DocChapter,
+ title: str,
+ text: str,
+ submodule,
+ installed: bool = True,
+ ):
+ super().__init__(chapter, title, text, None, installed, False)
+ self.section = submodule
+
+ if MATHICS_DEBUG_DOC_BUILD:
+ print(" DEBUG Creating Guide Section", title)
+
+
+class DocPart:
+ """
+ Represents one of the main parts of the document. Parts
+ can be loaded from a mdoc file, generated automatically from
+ the docstrings of Builtin objects under `mathics.builtin`.
+ """
+
+ chapter_class = DocChapter
+
+ def __init__(self, documentation, title, is_reference=False):
+ self.documentation = documentation
+ self.title = title
+ self.chapters = []
+ self.chapters_by_slug = {}
+ self.is_reference = is_reference
+ self.is_appendix = False
+ self.slug = slugify(title)
+ documentation.parts_by_slug[self.slug] = self
+ if MATHICS_DEBUG_DOC_BUILD:
+ print("DEBUG Creating Part", title)
+
+ def __str__(self) -> str:
+ return f" Part {self.title}\n\n" + "\n\n".join(
+ str(chapter) for chapter in sorted_chapters(self.chapters)
+ )
+
+
+class Documentation:
+ """
+ `Documentation` describes an object containing the whole documentation system.
+ Documentation
+ |
+ +--------0> Parts
+ |
+ +-----0> Chapters
+ |
+ +-----0>Sections
+ | |
+ | +------0> SubSections
+ |
+ +---->0>GuideSections
+ |
+ +------0> SubSections
+
+ (with 0>) meaning "aggregation".
+
+ Each element contains a title, a collection of elements of the following class
+ in the hierarchy. Parts, Chapters, Guide Sections, Sections and SubSections contains a doc
+ attribute describing the content to be shown after the title, and before
+ the elements of the subsequent terms in the hierarchy.
+ """
+
+ def __init__(self, title: str = "Title", doc_dir: str = ""):
+ """
+ Parameters
+ ----------
+ title : str, optional
+ The title of the Documentation. The default is "Title".
+ doc_dir : str, optional
+ The path where the sources can be loaded. The default is "",
+ meaning that no sources must be loaded.
+ """
+ # This is a way to load the default classes
+ # without defining these attributes as class
+ # attributes.
+ self._set_classes()
+ self.appendix = []
+ self.doc_dir = doc_dir
+ self.parts = []
+ self.parts_by_slug = {}
+ self.title = title
+
+ def _set_classes(self):
+ """
+ Set the classes of the subelements. Must be overloaded
+ by the subclasses.
+ """
+ if not hasattr(self, "part_class"):
+ self.chapter_class = DocChapter
+ self.doc_class = DocumentationEntry
+ self.guide_section_class = DocGuideSection
+ self.part_class = DocPart
+ self.section_class = DocSection
+ self.subsection_class = DocSubsection
+
+ def __str__(self):
+ result = self.title + "\n" + len(self.title) * "~" + "\n"
+ return (
+ result + "\n\n".join([str(part) for part in self.parts]) + "\n" + 60 * "-"
+ )
+
+ def add_section(
+ self,
+ chapter,
+ section_name: str,
+ section_object,
+ operator,
+ is_guide: bool = False,
+ in_guide: bool = False,
+ summary_text="",
+ ):
+ """
+ Adds a DocSection or DocGuideSection
+ object to the chapter, a DocChapter object.
+ "section_object" is either a Python module or a Class object instance.
+ """
+ if section_object is not None:
+ required_libs = getattr(section_object, "requires", [])
+ installed = check_requires_list(required_libs) if required_libs else True
+ # FIXME add an additional mechanism in the module
+ # to allow a docstring and indicate it is not to go in the
+ # user manual
+ if not section_object.__doc__:
+ return None
+
+ installed = True
+
+ if is_guide:
+ section = self.guide_section_class(
+ chapter,
+ section_name,
+ section_object.__doc__,
+ section_object,
+ installed=installed,
+ )
+ chapter.guide_sections.append(section)
+ else:
+ section = self.section_class(
+ chapter,
+ section_name,
+ section_object.__doc__,
+ operator=operator,
+ installed=installed,
+ in_guide=in_guide,
+ summary_text=summary_text,
+ )
+ chapter.sections.append(section)
+
+ return section
+
+ def add_subsection(
+ self,
+ chapter,
+ section,
+ subsection_name: str,
+ instance,
+ operator=None,
+ in_guide=False,
+ ):
+ """
+ Append a subsection for ``instance`` into ``section.subsections``
+ """
+
+ required_libs = getattr(instance, "requires", [])
+ installed = check_requires_list(required_libs) if required_libs else True
+
+ # FIXME add an additional mechanism in the module
+ # to allow a docstring and indicate it is not to go in the
+ # user manual
+ if not instance.__doc__:
+ return
+ summary_text = (
+ instance.summary_text if hasattr(instance, "summary_text") else ""
+ )
+ subsection = self.subsection_class(
+ chapter,
+ section,
+ subsection_name,
+ instance.__doc__,
+ operator=operator,
+ installed=installed,
+ in_guide=in_guide,
+ summary_text=summary_text,
+ )
+ section.subsections.append(subsection)
+
+ def doc_part(self, title, start):
+ """
+ Build documentation structure for a "Part" - Reference
+ section or collection of Mathics3 Modules.
+ """
+
+ builtin_part = self.part_class(self, title, is_reference=start)
+ self.parts.append(builtin_part)
+
+ def get_part(self, part_slug):
+ """return a section from part key"""
+ return self.parts_by_slug.get(part_slug)
+
+ def get_chapter(self, part_slug, chapter_slug):
+ """return a section from part and chapter keys"""
+ part = self.parts_by_slug.get(part_slug)
+ if part:
+ return part.chapters_by_slug.get(chapter_slug)
+ return None
+
+ def get_section(self, part_slug, chapter_slug, section_slug):
+ """return a section from part, chapter and section keys"""
+ part = self.parts_by_slug.get(part_slug)
+ if part:
+ chapter = part.chapters_by_slug.get(chapter_slug)
+ if chapter:
+ return chapter.sections_by_slug.get(section_slug)
+ return None
+
+ def get_subsection(self, part_slug, chapter_slug, section_slug, subsection_slug):
+ """
+ return a section from part, chapter, section and subsection
+ keys
+ """
+ part = self.parts_by_slug.get(part_slug)
+ if part:
+ chapter = part.chapters_by_slug.get(chapter_slug)
+ if chapter:
+ section = chapter.sections_by_slug.get(section_slug)
+ if section:
+ return section.subsections_by_slug.get(subsection_slug)
+
+ return None
+
+ # FIXME: turn into a @property tests?
+ def get_tests(self) -> Iterator:
+ """
+ Returns a generator to extracts lists test objects.
+ """
+ for part in self.parts:
+ for chapter in sorted_chapters(part.chapters):
+ if MATHICS_DEBUG_TEST_CREATE:
+ print(f"DEBUG Gathering tests for Chapter {chapter.title}")
+
+ tests = chapter.doc.get_tests()
+ if tests:
+ yield Tests(part.title, chapter.title, "", tests)
+
+ for section in chapter.all_sections:
+ if section.installed:
+ if MATHICS_DEBUG_TEST_CREATE:
+ if isinstance(section, DocGuideSection):
+ print(
+ f"DEBUG Gathering tests for Guide Section {section.title}"
+ )
+ else:
+ print(
+ f"DEBUG Gathering tests for Section {section.title}"
+ )
+ tests = section.doc.get_tests()
+ if tests:
+ yield Tests(
+ part.title,
+ chapter.title,
+ section.title,
+ tests,
+ )
+
+ def load_documentation_sources(self):
+ """
+ Extract doctest data from various static XML-like doc files, Mathics3 Built-in functions
+ (inside mathics.builtin), and external Mathics3 Modules.
+
+ The extracted structure is stored in ``self``.
+ """
+ from mathics.doc.gather import gather_docs_from_files, gather_reference_part
+
+ assert (
+ len(self.parts) == 0
+ ), "The documentation must be empty to call this function."
+
+ gather_docs_from_files(self, self.doc_dir)
+ # Next extract data that has been loaded into Mathics3 when it runs.
+ # This is information from `mathics.builtin`.
+ # This is Part 2 of the documentation.
+
+ # Notice that in order to generate the documentation
+ # from the builtin classes, it is needed to call first to
+ # import_and_load_builtins()
+
+ for title, modules, builtins_by_module in [
+ (
+ "Reference of Built-in Symbols",
+ mathics3_builtins_modules,
+ global_builtins_by_module,
+ ),
+ (
+ MATHICS3_MODULES_TITLE,
+ pymathics_modules,
+ pymathics_builtins_by_module,
+ ),
+ ]:
+ self.parts.append(
+ gather_reference_part(self, title, modules, builtins_by_module)
+ )
+
+ # Finally, extract Appendix information. This include License text
+ # This is the final Part of the documentation.
+
+ for part in self.appendix:
+ self.parts.append(part)
+
+ def load_part_from_file(
+ self,
+ filename: str,
+ part_title: str,
+ chapter_order: int,
+ is_appendix: bool = False,
+ ) -> int:
+ """Load a markdown file as a part of the documentation"""
+ part = self.part_class(self, part_title)
+ with open(filename, "rb") as src_file:
+ text = src_file.read().decode("utf8")
+
+ text = filter_comments(text)
+ chapters = CHAPTER_RE.findall(text)
+ for chapter_title, text in chapters:
+ chapter = self.chapter_class(
+ part, chapter_title, chapter_order=chapter_order
+ )
+ chapter_order += 1
+ text += '<section title=""></section>'
+ section_texts = SECTION_RE.findall(text)
+ for pre_text, title, text in section_texts:
+ if title:
+ section = self.section_class(
+ chapter, title, text, operator=None, installed=True
+ )
+ chapter.sections.append(section)
+ subsections = SUBSECTION_RE.findall(text)
+ for subsection_title in subsections:
+ subsection = self.subsection_class(
+ chapter,
+ section,
+ subsection_title,
+ text,
+ )
+ section.subsections.append(subsection)
+ else:
+ section = None
+ if not chapter.doc:
+ chapter.doc = self.doc_class(pre_text, title, section)
+ pass
+
+ part.chapters.append(chapter)
+ if is_appendix:
+ part.is_appendix = True
+ self.appendix.append(part)
+ else:
+ self.parts.append(part)
+ return chapter_order
+
+
+class DocSubsection:
+ """An object for a Documented Subsection.
+ A Subsection is part of a Section.
+ """
+
+ def __init__(
+ self,
+ chapter,
+ section,
+ title,
+ text,
+ operator=None,
+ installed=True,
+ in_guide=False,
+ summary_text="",
+ ):
+ """
+ Information that goes into a subsection object. This can be a written text, or
+ text extracted from the docstring of a builtin module or class.
+
+ About some of the parameters...
+
+ Some subsections are contained in a grouping module and need special work to
+ get the grouping module name correct.
+
+ For example the Chapter "Colors" is a module so the docstring text for it is in
+ mathics/builtin/colors/__init__.py . In mathics/builtin/colors/named-colors.py we have
+ the "section" name for the class Red (the subsection) inside it.
+ """
+ title_summary_text = re.split(" -- ", title)
+ len_title = len(title_summary_text)
+ # We need the documentation object, to have access
+ # to the suitable subclass of DocumentationElement.
+ documentation = chapter.part.documentation
+
+ self.title = title_summary_text[0] if len_title > 0 else ""
+ self.summary_text = title_summary_text[1] if len_title > 1 else summary_text
+ self.doc = documentation.doc_class(text, title, None)
+ self.chapter = chapter
+ self.in_guide = in_guide
+ self.installed = installed
+ self.operator = operator
+
+ self.section = section
+ self.slug = slugify(title)
+ self.subsections = []
+ self.title = title
+ self.doc.set_parent_path(self)
+
+ # This smells wrong: Here a DocSection (a level in the documentation system)
+ # is mixed with a DocumentationEntry. `items` is an attribute of the
+ # `DocumentationEntry`, not of a Part / Chapter/ Section.
+ # The content of a subsection should be stored in self.doc,
+ # and the tests should set the rute (key) through self.doc.set_parent_doc
+ if in_guide:
+ # Tests haven't been picked out yet from the doc string yet.
+ # Gather them here.
+ self.items = self.doc.items
+
+ for item in self.items:
+ for test in item.get_tests():
+ assert test.key is not None
+ else:
+ self.items = []
+
+ if text.count("<dl>") != text.count("</dl>"):
+ raise ValueError(
+ "Missing opening or closing <dl> tag in "
+ "{} documentation".format(title)
+ )
+ self.section.subsections_by_slug[self.slug] = self
+
+ if MATHICS_DEBUG_DOC_BUILD:
+ print(" DEBUG Creating Subsection", title)
+
+ def __str__(self) -> str:
+ return f"=== {self.title} ===\n{self.doc}"
+
+ @property
+ def parent(self):
+ """the chapter where the section is"""
+ return self.section
+
+ @parent.setter
+ def parent(self, value):
+ raise TypeError("parent is a read-only property")
+
+ def get_tests(self):
+ """yield tests"""
+ if self.installed:
+ for test in self.doc.get_tests():
+ yield test
+
+
+class MathicsMainDocumentation(Documentation):
+ """
+ MathicsMainDocumentation specializes ``Documentation`` by providing the attributes
+ and methods needed to generate the documentation from the Mathics library.
+
+ The parts of the documentation are loaded from the Markdown files contained
+ in the path specified by ``self.doc_dir``. Files with names starting in numbers
+ are considered parts of the main text, while those that starts with other characters
+ are considered as appendix parts.
+
+ In addition to the parts loaded from markdown files, a ``Reference of Builtin-Symbols`` part
+ and a part for the loaded Pymathics modules are automatically generated.
+
+ In the ``Reference of Built-in Symbols`` tom-level modules and files in ``mathics.builtin``
+ are associated to Chapters. For single file submodules (like ``mathics.builtin.procedure``)
+ The chapter contains a Section for each Symbol in the module. For sub-packages
+ (like ``mathics.builtin.arithmetic``) sections are given by the sub-module files,
+ and the symbols in these sub-packages defines the Subsections. ``__init__.py`` in
+ subpackages are associated to GuideSections.
+
+ In a similar way, in the ``Pymathics`` part, each ``pymathics`` module defines a Chapter,
+ files in the module defines Sections, and Symbols defines Subsections.
+
+
+ ``MathicsMainDocumentation`` is also used for creating test data and saving it to a
+ Python Pickle file and running tests that appear in the documentation (doctests).
+
+ There are other classes DjangoMathicsDocumentation and LaTeXMathicsDocumentation
+ that format the data accumulated here. In fact I think those can sort of serve
+ instead of this.
+
+ """
+
+ def __init__(self):
+ super().__init__(title="Mathics Main Documentation", doc_dir=settings.DOC_DIR)
+ self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL
+ self.pymathics_doc_loaded = False
+ self.doc_data_file = settings.get_doctest_latex_data_path(
+ should_be_readable=True
+ )
+
+ def gather_doctest_data(self):
+ """
+ Populates the documentatation.
+ (deprecated)
+ """
+ logging.warning(
+ "gather_doctest_data is deprecated. Use load_documentation_sources"
+ )
+ return self.load_documentation_sources()
+
+
+def sorted_chapters(chapters: List[DocChapter]) -> List[DocChapter]:
+ """Return chapters sorted by title"""
+ return sorted(
+ chapters,
+ key=lambda chapter: str(chapter.sort_order)
+ if chapter.sort_order is not None
+ else chapter.title,
+ )
+
+
+def sorted_modules(modules) -> list:
+ """Return modules sorted by the ``sort_order`` attribute if that
+ exists, or the module's name if not."""
+ return sorted(
+ modules,
+ key=lambda module: module.sort_order
+ if hasattr(module, "sort_order")
+ else module.__name__,
+ )
diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py
index f8d5aa873..e30d897b0 100644
--- a/mathics/docpipeline.py
+++ b/mathics/docpipeline.py
@@ -27,12 +27,14 @@
from mathics.core.parser import MathicsSingleLineFeeder
from mathics.doc.common_doc import DocGuideSection, DocSection, MathicsMainDocumentation
from mathics.doc.doc_entries import DocTest, DocTests
-from mathics.doc.utils import load_doctest_data, print_and_log
+from mathics.doc.utils import load_doctest_data, print_and_log, slugify
from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule
from mathics.timing import show_lru_cache_statistics
class TestOutput(Output):
+ """Output class for tests"""
+
def max_stored_size(self, _):
return None
@@ -171,7 +173,10 @@ def fail(why):
if not output_ok:
return fail(
"Output:\n%s\nWanted:\n%s"
- % ("\n".join(str(o) for o in out), "\n".join(str(o) for o in wanted_out))
+ % (
+ "\n".join(str(o) for o in out),
+ "\n".join(str(o) for o in wanted_out),
+ )
)
return True
@@ -192,7 +197,10 @@ def create_output(tests, doctest_data, output_format="latex"):
continue
key = test.key
evaluation = Evaluation(
- DEFINITIONS, format=output_format, catch_interrupt=True, output=TestOutput()
+ DEFINITIONS,
+ format=output_format,
+ catch_interrupt=True,
+ output=TestOutput(),
)
try:
result = evaluation.parse_evaluate(test.test)
@@ -233,8 +241,8 @@ def load_pymathics_modules(module_names: set):
except PyMathicsLoadException:
print(f"Python module {module_name} is not a Mathics3 module.")
- except Exception as e:
- print(f"Python import errors with: {e}.")
+ except Exception as exc:
+ print(f"Python import errors with: {exc}.")
else:
print(f"Mathics3 Module {module_name} loaded")
loaded_modules.append(module_name)
@@ -261,7 +269,8 @@ def show_test_summary(
print()
if total == 0:
print_and_log(
- LOGFILE, f"No {entity_name} found with a name in: {entities_searched}."
+ LOGFILE,
+ f"No {entity_name} found with a name in: {entities_searched}.",
)
if "MATHICS_DEBUG_TEST_CREATE" not in os.environ:
print(f"Set environment MATHICS_DEBUG_TEST_CREATE to see {entity_name}.")
@@ -269,14 +278,42 @@ def show_test_summary(
print(SEP)
if not generate_output:
print_and_log(
- LOGFILE, f"""{failed} test{'s' if failed != 1 else ''} failed."""
+ LOGFILE,
+ f"""{failed} test{'s' if failed != 1 else ''} failed.""",
)
else:
print_and_log(LOGFILE, "All tests passed.")
if generate_output and (failed == 0 or keep_going):
save_doctest_data(output_data)
- return
+
+
+def section_tests_iterator(section, include_subsections=None):
+ """
+ Iterator over tests in a section.
+ A section contains tests in its documentation entry,
+ in the head of the chapter and in its subsections.
+ This function is a generator of all these tests.
+
+ Before yielding a test from a documentation entry,
+ the user definitions are reset.
+ """
+ chapter = section.chapter
+ subsections = [section]
+ if chapter.doc:
+ subsections = [chapter.doc] + subsections
+ if section.subsections:
+ subsections = subsections + section.subsections
+
+ for subsection in subsections:
+ if (
+ include_subsections is not None
+ and subsection.title not in include_subsections
+ ):
+ continue
+ DEFINITIONS.reset_user_definitions()
+ for test in subsection.get_tests():
+ yield test
#
@@ -294,7 +331,7 @@ def test_section_in_chapter(
start_at: int = 0,
skipped: int = 0,
max_tests: int = MAX_TESTS,
-) -> Tuple[int, int, list]:
+) -> Tuple[int, int, list, set]:
"""
Runs a tests for section ``section`` under a chapter or guide section.
Note that both of these contain a collection of section tests underneath.
@@ -306,8 +343,8 @@ def test_section_in_chapter(
If ``stop_on_failure`` is true then the remaining tests in a section are skipped when a test
fails.
"""
+ failed_sections = set()
section_name = section.title
-
# Start out assuming all subsections will be tested
include_subsections = None
@@ -319,67 +356,63 @@ def test_section_in_chapter(
chapter_name = chapter.title
part_name = chapter.part.title
index = 0
- if len(section.subsections) > 0:
- subsections = section.subsections
- else:
- subsections = [section]
-
+ subsections = [section]
if chapter.doc:
subsections = [chapter.doc] + subsections
+ if section.subsections:
+ subsections = subsections + section.subsections
+
+ for test in section_tests_iterator(section, include_subsections):
+ # Get key dropping off test index number
+ key = list(test.key)[1:-1]
+ if prev_key != key:
+ prev_key = key
+ section_name_for_print = " / ".join(key)
+ if quiet:
+ # We don't print with stars inside in test_case(), so print here.
+ print(f"Testing section: {section_name_for_print}")
+ index = 0
+ else:
+ # Null out section name, so that on the next iteration we do not print a section header
+ # in test_case().
+ section_name_for_print = ""
- for subsection in subsections:
- if (
- include_subsections is not None
- and subsection.title not in include_subsections
- ):
- continue
+ tests = test.tests if isinstance(test, DocTests) else [test]
- DEFINITIONS.reset_user_definitions()
- for test in subsection.get_tests():
- # Get key dropping off test index number
- key = list(test.key)[1:-1]
- if prev_key != key:
- prev_key = key
- section_name_for_print = " / ".join(key)
- if quiet:
- # We don't print with stars inside in test_case(), so print here.
- print(f"Testing section: {section_name_for_print}")
- index = 0
- else:
- # Null out section name, so that on the next iteration we do not print a section header
- # in test_case().
- section_name_for_print = ""
-
- tests = test.tests if isinstance(test, DocTests) else [test]
-
- for doctest in tests:
- if doctest.ignore:
- continue
+ for doctest in tests:
+ if doctest.ignore:
+ continue
- index += 1
- total += 1
- if index < start_at:
- skipped += 1
- continue
+ index += 1
+ total += 1
+ if total > max_tests:
+ return total, failed, prev_key, failed_sections
+ if index < start_at:
+ skipped += 1
+ continue
- if not test_case(
- doctest,
- total,
- index,
- quiet=quiet,
- section_name=section_name,
- section_for_print=section_name_for_print,
- chapter_name=chapter_name,
- part=part_name,
- ):
- failed += 1
- if stop_on_failure:
- break
- # If failed, do not continue with the other subsections.
- if failed and stop_on_failure:
- break
+ if not test_case(
+ doctest,
+ total,
+ index,
+ quiet=quiet,
+ section_name=section_name,
+ section_for_print=section_name_for_print,
+ chapter_name=chapter_name,
+ part=part_name,
+ ):
+ failed += 1
+ failed_sections.add(
+ (
+ part_name,
+ chapter_name,
+ key[-1],
+ )
+ )
+ if stop_on_failure:
+ return total, failed, prev_key, failed_sections
- return total, failed, prev_key
+ return total, failed, prev_key, failed_sections
# When 3.8 is base, the below can be a Literal type.
@@ -439,14 +472,16 @@ def test_tests(
keep_going: bool = False,
) -> Tuple[int, int, int, set, int]:
"""
- Runs a group of related tests, ``Tests`` provided that the section is not listed in ``excludes`` and
- the global test count given in ``index`` is not before ``start_at``.
+ Runs a group of related tests, ``Tests`` provided that the section is not
+ listed in ``excludes`` and the global test count given in ``index`` is not
+ before ``start_at``.
- Tests are from a section or subsection (when the section is a guide section),
+ Tests are from a section or subsection (when the section is a guide
+ section). If ``quiet`` is True, the progress and results of the tests
+ are shown.
- If ``quiet`` is True, the progress and results of the tests are shown.
-
- ``index`` has the current count. We will stop on the first failure if ``stop_on_failure`` is true.
+ ``index`` has the current count. We will stop on the first failure
+ if ``stop_on_failure`` is true.
"""
@@ -467,6 +502,24 @@ def test_tests(
if (output_data, names) == INVALID_TEST_GROUP_SETUP:
return total, failed, skipped, failed_symbols, index
+ def show_and_return():
+ """Show the resume and build the tuple to return"""
+ show_test_summary(
+ total,
+ failed,
+ "chapters",
+ names,
+ keep_going,
+ generate_output,
+ output_data,
+ )
+
+ if generate_output and (failed == 0 or keep_going):
+ save_doctest_data(output_data)
+
+ return total, failed, skipped, failed_symbols, index
+
+ # Loop over the whole documentation.
for part in DOCUMENTATION.parts:
for chapter in part.chapters:
for section in chapter.all_sections:
@@ -475,11 +528,12 @@ def test_tests(
continue
if total >= max_tests:
- break
+ return show_and_return()
(
total,
failed,
prev_key,
+ new_failed_symbols,
) = test_section_in_chapter(
section,
total,
@@ -490,30 +544,15 @@ def test_tests(
start_at=start_at,
max_tests=max_tests,
)
- if failed and stop_on_failure:
- break
+ if failed:
+ failed_symbols.update(new_failed_symbols)
+ if stop_on_failure:
+ return show_and_return()
else:
if generate_output:
- create_output(section.doc.get_tests(), output_data)
- if failed and stop_on_failure:
- break
- if failed and stop_on_failure:
- break
-
- show_test_summary(
- total,
- failed,
- "chapters",
- names,
- keep_going,
- generate_output,
- output_data,
- )
-
- if generate_output and (failed == 0 or keep_going):
- save_doctest_data(output_data)
+ create_output(section_tests_iterator(section), output_data)
- return total, failed, skipped, failed_symbols, index
+ return show_and_return()
def test_chapters(
@@ -535,6 +574,7 @@ def test_chapters(
fails.
"""
failed = total = 0
+ failed_symbols = set()
output_data, chapter_names = validate_group_setup(
include_chapters, "chapters", reload
@@ -543,20 +583,32 @@ def test_chapters(
return total
prev_key = []
- seen_chapters = set()
- for part in DOCUMENTATION.parts:
- for chapter in part.chapters:
- chapter_name = chapter.title
- if chapter_name not in include_chapters:
- continue
- seen_chapters.add(chapter_name)
+ def show_and_return():
+ """Show the resume and return"""
+ show_test_summary(
+ total,
+ failed,
+ "chapters",
+ chapter_names,
+ keep_going,
+ generate_output,
+ output_data,
+ )
+ return total
+ for chapter_name in include_chapters:
+ chapter_slug = slugify(chapter_name)
+ for part in DOCUMENTATION.parts:
+ chapter = part.chapters_by_slug.get(chapter_slug, None)
+ if chapter is None:
+ continue
for section in chapter.all_sections:
(
total,
failed,
prev_key,
+ failed_symbols,
) = test_section_in_chapter(
section,
total,
@@ -569,23 +621,8 @@ def test_chapters(
)
if generate_output and failed == 0:
create_output(section.doc.get_tests(), output_data)
- pass
- pass
- # Shortcut: if we already pass through all the
- # include_chapters, break the loop
- if seen_chapters == include_chapters:
- break
- show_test_summary(
- total,
- failed,
- "chapters",
- chapter_names,
- keep_going,
- generate_output,
- output_data,
- )
- return total
+ return show_and_return()
def test_sections(
@@ -621,6 +658,18 @@ def test_sections(
section_name_for_finish = None
prev_key = []
+ def show_and_return():
+ show_test_summary(
+ total,
+ failed,
+ "sections",
+ section_names,
+ keep_going,
+ generate_output,
+ output_data,
+ )
+ return total
+
for part in DOCUMENTATION.parts:
for chapter in part.chapters:
for section in chapter.all_sections:
@@ -628,6 +677,7 @@ def test_sections(
total,
failed,
prev_key,
+ failed_symbols,
) = test_section_in_chapter(
section=section,
total=total,
@@ -640,7 +690,6 @@ def test_sections(
if generate_output and failed == 0:
create_output(section.doc.get_tests(), output_data)
- pass
if last_section_name != section_name_for_finish:
if seen_sections == include_sections:
@@ -649,21 +698,11 @@ def test_sections(
if section_name_for_finish in include_sections:
seen_sections.add(section_name_for_finish)
last_section_name = section_name_for_finish
- pass
+
if seen_last_section:
- break
- pass
-
- show_test_summary(
- total,
- failed,
- "sections",
- section_names,
- keep_going,
- generate_output,
- output_data,
- )
- return total
+ return show_and_return()
+
+ return show_and_return()
def test_all(
@@ -676,6 +715,9 @@ def test_all(
doc_even_if_error=False,
excludes: set = set(),
) -> int:
+ """
+ Run all the tests in the documentation.
+ """
if not quiet:
print(f"Testing {version_string}")
@@ -730,7 +772,8 @@ def test_all(
if failed_symbols:
if stop_on_failure:
print_and_log(
- LOGFILE, "(not all tests are accounted for due to --stop-on-failure)"
+ LOGFILE,
+ "(not all tests are accounted for due to --stop-on-failure)",
)
print_and_log(LOGFILE, "Failed:")
for part, chapter, section in sorted(failed_symbols):
@@ -762,10 +805,18 @@ def save_doctest_data(output_data: Dict[tuple, dict]):
* test number
and the value is a dictionary of a Result.getdata() dictionary.
"""
+ if len(output_data) == 0:
+ print("output data is empty")
+ return
+ print("saving", len(output_data), "entries")
+ print(output_data.keys())
doctest_latex_data_path = settings.get_doctest_latex_data_path(
should_be_readable=False, create_parent=True
)
print(f"Writing internal document data to {doctest_latex_data_path}")
+ if len(output_data) == 0:
+ print("output data is empty")
+ return
i = 0
for key in output_data:
i = i + 1
@@ -785,8 +836,12 @@ def write_doctest_data(quiet=False, reload=False):
print(f"Extracting internal doc data for {version_string}")
print("This may take a while...")
+ doctest_latex_data_path = settings.get_doctest_latex_data_path(
+ should_be_readable=False, create_parent=True
+ )
+
try:
- output_data = load_doctest_data() if reload else {}
+ output_data = load_doctest_data(doctest_latex_data_path) if reload else {}
for tests in DOCUMENTATION.get_tests():
create_output(tests, output_data)
except KeyboardInterrupt:
@@ -798,6 +853,7 @@ def write_doctest_data(quiet=False, reload=False):
def main():
+ """main"""
global DEFINITIONS
global LOGFILE
global CHECK_PARTIAL_ELAPSED_TIME
@@ -810,7 +866,10 @@ def main():
"--help", "-h", help="show this help message and exit", action="help"
)
parser.add_argument(
- "--version", "-v", action="version", version="%(prog)s " + mathics.__version__
+ "--version",
+ "-v",
+ action="version",
+ version="%(prog)s " + mathics.__version__,
)
parser.add_argument(
"--chapters",
@@ -872,7 +931,10 @@ def main():
"--doc-only",
dest="doc_only",
action="store_true",
- help="generate pickled internal document data without running tests; Can't be used with --section or --reload.",
+ help=(
+ "generate pickled internal document data without running tests; "
+ "Can't be used with --section or --reload."
+ ),
)
parser.add_argument(
"--reload",
@@ -882,7 +944,11 @@ def main():
help="reload pickled internal document data, before possibly adding to it",
)
parser.add_argument(
- "--quiet", "-q", dest="quiet", action="store_true", help="hide passed tests"
+ "--quiet",
+ "-q",
+ dest="quiet",
+ action="store_true",
+ help="hide passed tests",
)
parser.add_argument(
"--keep-going",
@@ -915,6 +981,7 @@ def main():
action="store_true",
help="print cache statistics",
)
+ global DOCUMENTATION
global LOGFILE
args = parser.parse_args()
@@ -926,14 +993,12 @@ def main():
if args.logfilename:
LOGFILE = open(args.logfilename, "wt")
- global DOCUMENTATION
- DOCUMENTATION = MathicsMainDocumentation()
-
# LoadModule Mathics3 modules
if args.pymathics:
required_modules = set(args.pymathics.split(","))
load_pymathics_modules(required_modules)
+ DOCUMENTATION = MathicsMainDocumentation()
DOCUMENTATION.load_documentation_sources()
start_time = None
@@ -982,8 +1047,6 @@ def main():
doc_even_if_error=args.keep_going,
excludes=excludes,
)
- pass
- pass
if total > 0 and start_time is not None:
end_time = datetime.now()
diff --git a/test/consistency-and-style/test_summary_text.py b/test/consistency-and-style/test_summary_text.py
index e27d4c79c..18e97afa5 100644
--- a/test/consistency-and-style/test_summary_text.py
+++ b/test/consistency-and-style/test_summary_text.py
@@ -9,7 +9,7 @@
from mathics import __file__ as mathics_initfile_path
from mathics.core.builtin import Builtin
from mathics.core.load_builtin import name_is_builtin_symbol
-from mathics.doc.common_doc import skip_doc
+from mathics.doc.gather import skip_doc
# Get file system path name for mathics.builtin
mathics_path = osp.dirname(mathics_initfile_path)
diff --git a/test/doc/__init__.py b/test/doc/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/doc/test_common.py b/test/doc/test_common.py
index 8d7ff17e7..66a25c765 100644
--- a/test/doc/test_common.py
+++ b/test/doc/test_common.py
@@ -217,3 +217,30 @@ def test_load_mathics_documentation():
for subsection in section.subsections:
assert subsection.title not in visited_subsections
visited_subsections.add(subsection.title)
+
+
+def test_doc_parser():
+ for input_str, output_str in (
+ ["", "<imgpng src='figure.png' title='figure'>"],
+ [
+ "{#figure-label}",
+ "<imgpng src='figure.png' title='figure' label='figure-label'>",
+ ],
+ [
+ ("""\n`` python\ndef f(x):\n g[i](x)\n""" """ return x + 2\n``\n"""),
+ """<python>def f(x):\n g[i](x)\n return x + 2\n</python>""",
+ ],
+ ["[url de destino](/doc/algo)", "<url>:url de destino:/doc/algo</url>"],
+ ):
+ result = parse_docstring_to_DocumentationEntry_items(
+ input_str,
+ DocTests,
+ DocTest,
+ DocText,
+ (
+ "part example",
+ "chapter example",
+ "section example",
+ ),
+ )[0].text
+ assert result == output_str
diff --git a/test/doc/test_doctests.py b/test/doc/test_doctests.py
new file mode 100644
index 000000000..cee480a85
--- /dev/null
+++ b/test/doc/test_doctests.py
@@ -0,0 +1,112 @@
+"""
+Pytests for the documentation system. Basic functions and classes.
+"""
+import os.path as osp
+
+from mathics.core.evaluation import Message, Print
+from mathics.core.load_builtin import import_and_load_builtins
+from mathics.doc.common_doc import (
+ DocChapter,
+ DocPart,
+ DocSection,
+ Documentation,
+ MathicsMainDocumentation,
+)
+from mathics.doc.doc_entries import (
+ DocTest,
+ DocTests,
+ DocText,
+ DocumentationEntry,
+ parse_docstring_to_DocumentationEntry_items,
+)
+from mathics.settings import DOC_DIR
+
+import_and_load_builtins()
+DOCUMENTATION = MathicsMainDocumentation()
+DOCUMENTATION.load_documentation_sources()
+
+
+def test_load_doctests():
+ # there are in master 3959 tests...
+ all_the_tests = tuple((tests for tests in DOCUMENTATION.get_tests()))
+ visited_positions = set()
+ # Check that there are not dupliceted entries
+ for tests in all_the_tests:
+ position = (tests.part, tests.chapter, tests.section)
+ print(position)
+ assert position not in visited_positions
+ visited_positions.add(position)
+
+
+def test_create_doctest():
+ """initializing DocTest"""
+
+ key = (
+ "Part title",
+ "Chapter Title",
+ "Section Title",
+ )
+ test_cases = [
+ {
+ "test": [">", "2+2", "\n = 4"],
+ "properties": {
+ "private": False,
+ "ignore": False,
+ "result": "4",
+ "outs": [],
+ "key": key + (1,),
+ },
+ },
+ {
+ "test": ["#", "2+2", "\n = 4"],
+ "properties": {
+ "private": True,
+ "ignore": False,
+ "result": "4",
+ "outs": [],
+ "key": key + (1,),
+ },
+ },
+ {
+ "test": ["S", "2+2", "\n = 4"],
+ "properties": {
+ "private": False,
+ "ignore": False,
+ "result": "4",
+ "outs": [],
+ "key": key + (1,),
+ },
+ },
+ {
+ "test": ["X", 'Print["Hola"]', "| Hola"],
+ "properties": {
+ "private": False,
+ "ignore": True,
+ "result": None,
+ "outs": [Print("Hola")],
+ "key": key + (1,),
+ },
+ },
+ {
+ "test": [
+ ">",
+ "1 / 0",
+ "\n : Infinite expression 1 / 0 encountered.\n ComplexInfinity",
+ ],
+ "properties": {
+ "private": False,
+ "ignore": False,
+ "result": None,
+ "outs": [
+ Message(
+ symbol="", text="Infinite expression 1 / 0 encountered.", tag=""
+ )
+ ],
+ "key": key + (1,),
+ },
+ },
+ ]
+ for index, test_case in enumerate(test_cases):
+ doctest = DocTest(1, test_case["test"], key)
+ for property_key, value in test_case["properties"].items():
+ assert getattr(doctest, property_key) == value
diff --git a/test/doc/test_latex.py b/test/doc/test_latex.py
index 4e4e9a1cc..e98043836 100644
--- a/test/doc/test_latex.py
+++ b/test/doc/test_latex.py
@@ -90,7 +90,7 @@ def test_load_latex_documentation():
).strip() == "Let's sketch the function\n\\begin{tests}"
assert (
first_section.latex(doc_data)[:30]
- ).strip() == "\\section*{Curve Sketching}{}"
+ ).strip() == "\\section{Curve Sketching}{}"
assert (
third_chapter.latex(doc_data)[:38]
).strip() == "\\chapter{Further Tutorial Examples}"
@@ -102,10 +102,11 @@ def test_chapter():
chapter = part.chapters_by_slug["testing-expressions"]
print(chapter.sections_by_slug.keys())
section = chapter.sections_by_slug["numerical-properties"]
- latex_section_head = section.latex({})[:63].strip()
- assert (
- latex_section_head
- == "\section*{Numerical Properties}{\index{Numerical Properties}}"
+ latex_section_head = section.latex({})[:90].strip()
+ assert latex_section_head == (
+ "\\section{Numerical Properties}\n"
+ "\\sectionstart\n\n\n\n"
+ "\\subsection*{CoprimeQ}\index{CoprimeQ}"
)
print(60 * "@")
latex_chapter = chapter.latex({}, quiet=False)