From 25e877933b6b22593cd6216396f2ffe9762cda4e Mon Sep 17 00:00:00 2001 From: Juan Mauricio Matera Date: Sat, 23 Mar 2024 21:24:06 -0300 Subject: [PATCH 1/2] Split doc test classes (#1021) Another thing that have been waiting to do: splitting `mathics.doc.common_doc` into simpler, more specific pieces. I hope that this is going to help in the next steps (sorting the modules, accessing specific test cases in docpipeline, etc) This also includes * Split `mathics.doc.common_doc` into simpler more specific pieces: the structure ("Documentation/Chapter/Section..." in `common_doc` and the documentation entries itself (`doc_entries`) ). * DRY code in `docpipeline`. * Run the tests in the Chapter documentation. * Fixes some references in ImportExport documentation. * Add and improve some docstrings and comments. * makes that the key attribute in `mathics.doc.common.DocTest` being more deterministic. * DRY __init__ routines in latex_doc subclasses. --- mathics/builtin/files_io/importexport.py | 8 +- mathics/doc/common_doc.py | 635 ++++------------------- mathics/doc/doc_entries.py | 548 +++++++++++++++++++ mathics/doc/latex_doc.py | 135 +---- mathics/docpipeline.py | 251 ++++----- test/doc/test_common.py | 6 +- test/doc/test_latex.py | 2 +- 7 files changed, 790 insertions(+), 795 deletions(-) create mode 100644 mathics/doc/doc_entries.py mode change 100755 => 100644 mathics/docpipeline.py diff --git a/mathics/builtin/files_io/importexport.py b/mathics/builtin/files_io/importexport.py index fff53163a..1eecd4bed 100644 --- a/mathics/builtin/files_io/importexport.py +++ b/mathics/builtin/files_io/importexport.py @@ -5,16 +5,16 @@ Many kinds data formats can be read into \Mathics. Variable :$ExportFormats: -/doc/reference-of-built-in-symbols/importing-and-exporting/$exportformats \ +/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/$exportformats \ contains a list of file formats that are supported by :Export: -/doc/reference-of-built-in-symbols/importing-and-exporting/export, \ +/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/export, \ while :$ImportFormats: -/doc/reference-of-built-in-symbols/importing-and-exporting/$importformats \ +/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/$importformats \ does the corresponding thing for :Import: -/doc/reference-of-built-in-symbols/importing-and-exporting/import. +/doc/reference-of-built-in-symbols/inputoutput-files-and-filesystem/importing-and-exporting/import. """ import base64 diff --git a/mathics/doc/common_doc.py b/mathics/doc/common_doc.py index e772a1035..0a3e041af 100644 --- a/mathics/doc/common_doc.py +++ b/mathics/doc/common_doc.py @@ -21,116 +21,41 @@ As with reading in data, final assembly to a LaTeX file or running documentation tests is done elsewhere. - FIXME: This code should be replaced by Sphinx and autodoc. -Things are such a mess, that it is too difficult to contemplate this right now. Also there -higher-priority flaws that are more more pressing. -In the shorter, we might we move code for extracting printing to a separate package. +Things are such a mess, that it is too difficult to contemplate this right now. +Also there higher-priority flaws that are more more pressing. +In the shorter, we might we move code for extracting printing to a +separate package. """ -import importlib import logging import os.path as osp import pkgutil import re -from os import environ, getenv, listdir +from os import environ, listdir from types import ModuleType -from typing import Callable, Iterator, List, Optional, Tuple +from typing import Iterator, List, Optional, Tuple from mathics import settings from mathics.core.builtin import check_requires_list -from mathics.core.evaluation import Message, Print from mathics.core.load_builtin import ( builtins_by_module as global_builtins_by_module, mathics3_builtins_modules, ) from mathics.core.util import IS_PYPY +from mathics.doc.doc_entries import ( + DocumentationEntry, + Tests, + filter_comments, + parse_docstring_to_DocumentationEntry_items, +) from mathics.doc.utils import slugify from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules -# These are all the XML/HTML-like tags that documentation supports. -ALLOWED_TAGS = ( - "dl", - "dd", - "dt", - "em", - "url", - "ul", - "i", - "ol", - "li", - "con", - "console", - "img", - "imgpng", - "ref", - "subsection", -) -ALLOWED_TAGS_RE = dict( - (allowed, re.compile("<(%s.*?)>" % allowed)) for allowed in ALLOWED_TAGS -) - -# This string is used, so we can indicate a trailing blank at the end of a line by -# adding this string to the end of the line which gets stripped off. -# Some editors and formatters like to strip off trailing blanks at the ends of lines. -END_LINE_SENTINAL = "#<--#" - -# The regular expressions below (strings ending with _RE -# pull out information from docstring or text in a file. Ghetto parsing. - CHAPTER_RE = re.compile('(?s)(.*?)') -CONSOLE_RE = re.compile(r"(?s)<(?Pcon|console)>(?P.*?)") -DL_ITEM_RE = re.compile( - r"(?s)<(?Pd[td])>(?P.*?)(?:|)\s*(?:(?=)|$)" -) -DL_RE = re.compile(r"(?s)
(.*?)
") -HYPERTEXT_RE = re.compile( - r"(?s)<(?Pem|url)>(\s*:(?P.*?):\s*)?(?P.*?)" -) -IMG_PNG_RE = re.compile( - r'' -) -IMG_RE = re.compile( - r'' -) -# Preserve space before and after in-line code variables. -LATEX_RE = re.compile(r"(\s?)\$(\w+?)\$(\s?)") - -LIST_ITEM_RE = re.compile(r"(?s)
  • (.*?)(?:
    • |(?=
  • )|$)") -LIST_RE = re.compile(r"(?s)<(?Pul|ol)>(?P.*?)") -MATHICS_RE = re.compile(r"(?(.*?)") -QUOTATIONS_RE = re.compile(r"\"([\w\s,]*?)\"") -REF_RE = re.compile(r'') SECTION_RE = re.compile('(?s)(.*?)
    (.*?)
    ') -SPECIAL_COMMANDS = { - "LaTeX": (r"LaTeX", r"\LaTeX{}"), - "Mathematica": ( - r"Mathematica®", - r"\emph{Mathematica}\textregistered{}", - ), - "Mathics": (r"Mathics3", r"\emph{Mathics3}"), - "Mathics3": (r"Mathics3", r"\emph{Mathics3}"), - "Sage": (r"Sage", r"\emph{Sage}"), - "Wolfram": (r"Wolfram", r"\emph{Wolfram}"), - "skip": (r"

    ", r"\bigskip"), -} -SUBSECTION_END_RE = re.compile("") SUBSECTION_RE = re.compile('(?s)') -TESTCASE_RE = re.compile( - r"""(?mx)^ # re.MULTILINE (multi-line match) - # and re.VERBOSE (readable regular expressions - ((?:.|\n)*?) - ^\s+([>#SX])>[ ](.*) # test-code indicator - ((?:\n\s*(?:[:|=.][ ]|\.).*)*) # test-code results""" -) -TESTCASE_OUT_RE = re.compile(r"^\s*([:|=])(.*)$") - -# Used for getting test results by test expresson and chapter/section information. -test_result_map = {} - # Debug flags. # Set to True if want to follow the process @@ -160,9 +85,8 @@ def get_module_doc(module: ModuleType) -> Tuple[str, str]: title = doc.splitlines()[0] text = "\n".join(doc.splitlines()[1:]) else: - # FIXME: Extend me for Mathics3 modules. title = module.__name__ - for prefix in ("mathics.builtin.", "mathics.optional."): + for prefix in ("mathics.builtin.", "mathics.optional.", "pymathics."): if title.startswith(prefix): title = title[len(prefix) :] title = title.capitalize() @@ -170,54 +94,6 @@ def get_module_doc(module: ModuleType) -> Tuple[str, str]: return title, text -def get_results_by_test(test_expr: str, full_test_key: list, doc_data: dict) -> dict: - """ - Sometimes test numbering is off, either due to bugs or changes since the - data was read. - - Here, we compensate for this by looking up the test by its chapter and section name - portion stored in `full_test_key` along with the and the test expression data - stored in `test_expr`. - - This new key is looked up in `test_result_map` its value is returned. - - `doc_data` is only first time this is called to populate `test_result_map`. - """ - - # Strip off the test index form new key with this and the test string. - # Add to any existing value for that "result". This is now what we want to - # use as a tee in test_result_map to look for. - test_section = list(full_test_key)[:-1] - search_key = tuple(test_section) - - if not test_result_map: - # Populate test_result_map from doc_data - for key, result in doc_data.items(): - test_section = list(key)[:-1] - new_test_key = tuple(test_section) - next_result = test_result_map.get(new_test_key, None) - if next_result is None: - next_result = [result] - else: - next_result.append(result) - - test_result_map[new_test_key] = next_result - - results = test_result_map.get(search_key, None) - result = {} - if results: - for result_candidate in results: - if result_candidate["query"] == test_expr: - if result: - # Already found something - print(f"Warning, multiple results appear under {search_key}.") - return {} - else: - result = result_candidate - - return result - - def get_submodule_names(obj) -> list: """Many builtins are organized into modules which, from a documentation standpoint, are like Mathematica Online Guide Docs. @@ -253,14 +129,6 @@ def get_submodule_names(obj) -> list: return modpkgs -def filter_comments(doc: str) -> str: - """Remove docstring documentation comments. These are lines - that start with ##""" - return "\n".join( - line for line in doc.splitlines() if not line.lstrip().startswith("##") - ) - - def get_doc_name_from_module(module) -> str: """ Get the title associated to the module. @@ -278,35 +146,13 @@ def get_doc_name_from_module(module) -> str: return name -POST_SUBSTITUTION_TAG = "_POST_SUBSTITUTION%d_" - - -def pre_sub(regexp, text: str, repl_func): - post_substitutions = [] - - def repl_pre(match): - repl = repl_func(match) - index = len(post_substitutions) - post_substitutions.append(repl) - return POST_SUBSTITUTION_TAG % index - - text = regexp.sub(repl_pre, text) - - return text, post_substitutions - - -def post_sub(text: str, post_substitutions) -> str: - for index, sub in enumerate(post_substitutions): - text = text.replace(POST_SUBSTITUTION_TAG % index, sub) - return text - - def skip_doc(cls) -> bool: """Returns True if we should skip cls in docstring extraction.""" return cls.__name__.endswith("Box") or (hasattr(cls, "no_doc") and cls.no_doc) def skip_module_doc(module, must_be_skipped) -> bool: + """True if the module should not be included in the documentation""" return ( module.__doc__ is None or module in must_be_skipped @@ -316,176 +162,6 @@ def skip_module_doc(module, must_be_skipped) -> bool: ) -def parse_docstring_to_DocumentationEntry_items( - doc: str, - test_collection_constructor: Callable, - test_case_constructor: Callable, - text_constructor: Callable, - key_part=None, -) -> list: - """ - This parses string `doc` (using regular expressions) into Python objects. - test_collection_fn() is the class construtorto call to create an object for the - test collection. Each test is created via test_case_fn(). - Text within the test is stored via text_constructor. - """ - # Remove commented lines. - doc = filter_comments(doc).strip(r"\s") - - # Remove leading
    ...
    - # doc = DL_RE.sub("", doc) - - # pre-substitute Python code because it might contain tests - doc, post_substitutions = pre_sub( - PYTHON_RE, doc, lambda m: "%s" % m.group(1) - ) - - # HACK: Artificially construct a last testcase to get the "intertext" - # after the last (real) testcase. Ignore the test, of course. - doc += "\n >> test\n = test" - testcases = TESTCASE_RE.findall(doc) - - tests = None - items = [] - for index in range(len(testcases)): - testcase = list(testcases[index]) - text = testcase.pop(0).strip() - if text: - if tests is not None: - items.append(tests) - tests = None - text = post_sub(text, post_substitutions) - items.append(text_constructor(text)) - tests = None - if index < len(testcases) - 1: - test = test_case_constructor(index, testcase, key_part) - if tests is None: - tests = test_collection_constructor() - tests.tests.append(test) - - # If the last block in the loop was not a Text block, append the - # last set of tests. - if tests is not None: - items.append(tests) - tests = None - return items - - -class DocTest: - """ - Class to hold a single doctest. - - DocTest formatting rules: - - * `>>` Marks test case; it will also appear as part of - the documentation. - * `#>` Marks test private or one that does not appear as part of - the documentation. - * `X>` Shows the example in the docs, but disables testing the example. - * `S>` Shows the example in the docs, but disables testing if environment - variable SANDBOX is set. - * `=` Compares the result text. - * `:` Compares an (error) message. - `|` Prints output. - """ - - def __init__( - self, index: int, testcase: List[str], key_prefix: Optional[tuple] = None - ): - def strip_sentinal(line: str): - """Remove END_LINE_SENTINAL from the end of a line if it appears. - - Some editors like to strip blanks at the end of a line. - Since the line ends in END_LINE_SENTINAL which isn't blank, - any blanks that appear before will be preserved. - - Some tests require some lines to be blank or entry because - Mathics3 output can be that way - """ - if line.endswith(END_LINE_SENTINAL): - line = line[: -len(END_LINE_SENTINAL)] - - # Also remove any remaining trailing blanks since that - # seems *also* what we want to do. - return line.strip() - - self.index = index - self.outs = [] - self.result = None - - # Private test cases are executed, but NOT shown as part of the docs - self.private = testcase[0] == "#" - - # Ignored test cases are NOT executed, but shown as part of the docs - # Sandboxed test cases are NOT executed if environment SANDBOX is set - if testcase[0] == "X" or (testcase[0] == "S" and getenv("SANDBOX", False)): - self.ignore = True - # substitute '>' again so we get the correct formatting - testcase[0] = ">" - else: - self.ignore = False - - self.test = strip_sentinal(testcase[1]) - - self.key = None - if key_prefix: - self.key = tuple(key_prefix + (index,)) - - outs = testcase[2].splitlines() - for line in outs: - line = strip_sentinal(line) - if line: - if line.startswith("."): - text = line[1:] - if text.startswith(" "): - text = text[1:] - text = "\n" + text - if self.result is not None: - self.result += text - elif self.outs: - self.outs[-1].text += text - continue - - match = TESTCASE_OUT_RE.match(line) - if not match: - continue - symbol, text = match.group(1), match.group(2) - text = text.strip() - if symbol == "=": - self.result = text - elif symbol == ":": - out = Message("", "", text) - self.outs.append(out) - elif symbol == "|": - out = Print(text) - self.outs.append(out) - - def __str__(self) -> str: - return self.test - - -# Tests has to appear before Documentation which uses it. -# FIXME: Turn into a NamedTuple? Or combine with another class? -class Tests: - """ - A group of tests in the same section or subsection. - """ - - def __init__( - self, - part_name: str, - chapter_name: str, - section_name: str, - doctests: List[DocTest], - subsection_name: Optional[str] = None, - ): - self.part = part_name - self.chapter = chapter_name - self.section = section_name - self.subsection = subsection_name - self.tests = doctests - - # DocSection has to appear before DocGuideSection which uses it. class DocSection: """An object for a Documented Section. @@ -522,7 +198,10 @@ def __init__( # Needs to come after self.chapter is initialized since # DocumentationEntry uses self.chapter. - self.doc = DocumentationEntry(text, title, self) + # Notice that we need the documentation object, to have access + # to the suitable subclass of DocumentationElement. + documentation = self.chapter.part.doc + self.doc = documentation.doc_class(text, title, None).set_parent_path(self) chapter.sections_by_slug[self.slug] = self if MATHICS_DEBUG_DOC_BUILD: @@ -544,6 +223,14 @@ def get_tests(self): for test in self.doc.get_tests(): yield test + @property + def parent(self): + return self.chapter + + @parent.setter + def parent(self, value): + raise TypeError("parent is a read-only property") + # DocChapter has to appear before DocGuideSection which uses it. class DocChapter: @@ -561,6 +248,8 @@ def __init__(self, part, title, doc=None, chapter_order: Optional[int] = None): self.sections = [] self.sections_by_slug = {} self.sort_order = None + if doc: + self.doc.set_parent_path(self) part.chapters_by_slug[self.slug] = self @@ -585,6 +274,14 @@ def __str__(self) -> str: def all_sections(self): return sorted(self.sections + self.guide_sections) + @property + def parent(self): + return self.part + + @parent.setter + def parent(self, value): + raise TypeError("parent is a read-only property") + class DocGuideSection(DocSection): """An object for a Documented Guide Section. @@ -601,36 +298,24 @@ def __init__( submodule, installed: bool = True, ): - self.chapter = chapter - self.doc = DocumentationEntry(text, title, None) - self.in_guide = False - self.installed = installed + super().__init__(chapter, title, text, None, installed, False) self.section = submodule - self.slug = slugify(title) - self.subsections = [] - self.subsections_by_slug = {} - self.title = title - # FIXME: Sections never are operators. Subsections can have - # operators though. Fix up the view and searching code not to - # look for the operator field of a section. - self.operator = False - - if text.count("
    ") != text.count("
    "): - raise ValueError( - "Missing opening or closing
    tag in " - "{} documentation".format(title) - ) if MATHICS_DEBUG_DOC_BUILD: print(" DEBUG Creating Guide Section", title) - chapter.sections_by_slug[self.slug] = self # FIXME: turn into a @property tests? def get_tests(self): + """ + Tests included in a Guide. + """ # FIXME: The below is a little weird for Guide Sections. # Figure out how to make this clearer. # A guide section's subsection are Sections without the Guide. # it is *their* subsections where we generally find tests. + # + # Currently, this is not called in docpipeline or in making + # the LaTeX documentation. for section in self.subsections: if not section.installed: continue @@ -690,31 +375,6 @@ def __str__(self) -> str: ) -class DocTests: - """ - A bunch of consecutive ``DocTest`` objects extracted from a Builtin docstring. - """ - - def __init__(self): - self.tests = [] - self.text = "" - - def get_tests(self) -> list: - """ - Returns lists test objects. - """ - return self.tests - - def is_private(self) -> bool: - return all(test.private for test in self.tests) - - def __str__(self) -> str: - return "\n".join(str(test) for test in self.tests) - - def test_indices(self) -> List[int]: - return [test.index for test in self.tests] - - class Documentation: """ `Documentation` describes an object containing the whole documentation system. @@ -742,15 +402,25 @@ class Documentation: the elements of the subsequent terms in the hierarchy. """ - def __init__(self): + def __init__(self, title: str = "Title", doc_dir: str = ""): + """ + Parameters + ---------- + title : str, optional + The title of the Documentation. The default is "Title". + doc_dir : str, optional + The path where the sources can be loaded. The default is "", + meaning that no sources must be loaded. + """ # This is a way to load the default classes # without defining these attributes as class # attributes. self._set_classes() - self.parts = [] self.appendix = [] + self.doc_dir = doc_dir + self.parts = [] self.parts_by_slug = {} - self.title = "Title" + self.title = title def _set_classes(self): """ @@ -955,7 +625,7 @@ def doc_chapter(self, module, part, builtins_by_module) -> Optional[DocChapter]: guide_section.subsections.append(section) builtins = builtins_by_module.get(submodule.__name__, []) - subsections = [builtin for builtin in builtins] + subsections = list(builtins) for instance in subsections: if hasattr(instance, "no_doc") and instance.no_doc: continue @@ -985,6 +655,10 @@ def doc_chapter(self, module, part, builtins_by_module) -> Optional[DocChapter]: return chapter def doc_sections(self, sections, modules_seen, chapter): + """ + Load sections from a list of mathics builtins. + """ + for instance in sections: if instance not in modules_seen and ( not hasattr(instance, "no_doc") or not instance.no_doc @@ -1005,15 +679,18 @@ def doc_sections(self, sections, modules_seen, chapter): modules_seen.add(instance) def get_part(self, part_slug): + """return a section from part key""" return self.parts_by_slug.get(part_slug) def get_chapter(self, part_slug, chapter_slug): + """return a section from part and chapter keys""" part = self.parts_by_slug.get(part_slug) if part: return part.chapters_by_slug.get(chapter_slug) return None def get_section(self, part_slug, chapter_slug, section_slug): + """return a section from part, chapter and section keys""" part = self.parts_by_slug.get(part_slug) if part: chapter = part.chapters_by_slug.get(chapter_slug) @@ -1022,6 +699,10 @@ def get_section(self, part_slug, chapter_slug, section_slug): return None def get_subsection(self, part_slug, chapter_slug, section_slug, subsection_slug): + """ + return a section from part, chapter, section and subsection + keys + """ part = self.parts_by_slug.get(part_slug) if part: chapter = part.chapters_by_slug.get(chapter_slug) @@ -1087,14 +768,11 @@ def get_tests(self) -> Iterator: tests = section.doc.get_tests() if tests: yield Tests( - part.title, chapter.title, section.title, tests + part.title, + chapter.title, + section.title, + tests, ) - pass - pass - pass - pass - pass - pass return def load_documentation_sources(self): @@ -1162,18 +840,14 @@ def load_documentation_sources(self): for part in self.appendix: self.parts.append(part) - # Via the wanderings above, collect all tests that have been - # seen. - # - # Each test is accessble by its part + chapter + section and test number - # in that section. - for tests in self.get_tests(): - for test in tests.tests: - test.key = (tests.part, tests.chapter, tests.section, test.index) return def load_part_from_file( - self, filename: str, title: str, chapter_order: int, is_appendix: bool = False + self, + filename: str, + title: str, + chapter_order: int, + is_appendix: bool = False, ) -> int: """Load a markdown file as a part of the documentation""" part = self.part_class(self, title) @@ -1200,8 +874,6 @@ def load_part_from_file( text, ) section.subsections.append(subsection) - pass - pass else: section = None if not chapter.doc: @@ -1244,14 +916,17 @@ def __init__( For example the Chapter "Colors" is a module so the docstring text for it is in mathics/builtin/colors/__init__.py . In mathics/builtin/colors/named-colors.py we have - the "section" name for the class Read (the subsection) inside it. + the "section" name for the class Red (the subsection) inside it. """ title_summary_text = re.split(" -- ", title) n = len(title_summary_text) + # We need the documentation object, to have access + # to the suitable subclass of DocumentationElement. + documentation = chapter.part.doc + self.title = title_summary_text[0] if n > 0 else "" self.summary_text = title_summary_text[1] if n > 1 else summary_text - - self.doc = DocumentationEntry(text, title, section) + self.doc = documentation.doc_class(text, title, None) self.chapter = chapter self.in_guide = in_guide self.installed = installed @@ -1261,21 +936,21 @@ def __init__( self.slug = slugify(title) self.subsections = [] self.title = title + self.doc.set_parent_path(self) - if section: - chapter = section.chapter - part = chapter.part - # Note: we elide section.title - key_prefix = (part.title, chapter.title, title) - else: - key_prefix = None - + # This smells wrong: Here a DocSection (a level in the documentation system) + # is mixed with a DocumentationEntry. `items` is an attribute of the + # `DocumentationEntry`, not of a Part / Chapter/ Section. + # The content of a subsection should be stored in self.doc, + # and the tests should set the rute (key) through self.doc.set_parent_doc if in_guide: # Tests haven't been picked out yet from the doc string yet. # Gather them here. - self.items = parse_docstring_to_DocumentationEntry_items( - text, DocTests, DocTest, DocText, key_prefix - ) + self.items = self.doc.items + + for item in self.items: + for test in item.get_tests(): + assert test.key is not None else: self.items = [] @@ -1285,12 +960,21 @@ def __init__( "{} documentation".format(title) ) self.section.subsections_by_slug[self.slug] = self + if MATHICS_DEBUG_DOC_BUILD: print(" DEBUG Creating Subsection", title) def __str__(self) -> str: return f"=== {self.title} ===\n{self.doc}" + @property + def parent(self): + return self.section + + @parent.setter + def parent(self, value): + raise TypeError("parent is a read-only property") + def get_tests(self): """yield tests""" if self.installed: @@ -1332,133 +1016,24 @@ class MathicsMainDocumentation(Documentation): """ def __init__(self): - super().__init__() - - self.doc_dir = settings.DOC_DIR + super().__init__(title="Mathics Main Documentation", doc_dir=settings.DOC_DIR) self.doctest_latex_pcl_path = settings.DOCTEST_LATEX_DATA_PCL self.pymathics_doc_loaded = False self.doc_data_file = settings.get_doctest_latex_data_path( should_be_readable=True ) - self.title = "Mathics Main Documentation" def gather_doctest_data(self): """ Populates the documentatation. (deprecated) """ - logging.warn( + logging.warning( "gather_doctest_data is deprecated. Use load_documentation_sources" ) return self.load_documentation_sources() -class DocText: - """ - Class to hold some (non-test) text. - - Some of the kinds of tags you may find here are showin in global ALLOWED_TAGS. - Some text may be marked with surrounding "$" or "'". - - The code here however does not make use of any of the tagging. - - """ - - def __init__(self, text): - self.text = text - - def __str__(self) -> str: - return self.text - - def get_tests(self) -> list: - """ - Return tests in a DocText item - there never are any. - """ - return [] - - def is_private(self) -> bool: - return False - - def test_indices(self) -> List[int]: - return [] - - -# Former XMLDoc -class DocumentationEntry: - """ - A class to hold the content of a documentation entry, - in our internal markdown-like format data. - - Describes the contain of an entry in the documentation system, as a - sequence (list) of items of the clase `DocText` and `DocTests`. - ``DocText`` items contains an internal XML-like formatted text. ``DocTests`` entries - contain one or more `DocTest` element. - Each level of the Documentation hierarchy contains an XMLDoc, describing the - content after the title and before the elements of the next level. For example, - in ``DocChapter``, ``DocChapter.doc`` contains the text coming after the title - of the chapter, and before the sections in `DocChapter.sections`. - Specialized classes like LaTeXDoc or and DjangoDoc provide methods for - getting formatted output. For LaTeXDoc ``latex()`` is added while for - DjangoDoc ``html()`` is added - Mathics core also uses this in getting usage strings (`??`). - - """ - - def __init__(self, doc_str: str, title: str, section: Optional[DocSection] = None): - self._set_classes() - self.title = title - if section: - chapter = section.chapter - part = chapter.part - # Note: we elide section.title - key_prefix = (part.title, chapter.title, title) - else: - key_prefix = None - - self.rawdoc = doc_str - self.items = parse_docstring_to_DocumentationEntry_items( - self.rawdoc, - self.docTest_collection_class, - self.docTest_class, - self.docText_class, - key_prefix, - ) - - def _set_classes(self): - """ - Tells to the initializator the classes to be used to build the items. - This must be overloaded by the daughter classes. - """ - if not hasattr(self, "docTest_collection_class"): - self.docTest_collection_class = DocTests - self.docTest_class = DocTest - self.docText_class = DocText - - def __str__(self) -> str: - return "\n\n".join(str(item) for item in self.items) - - def text(self) -> str: - # used for introspection - # TODO parse XML and pretty print - # HACK - item = str(self.items[0]) - item = "\n".join(line.strip() for line in item.split("\n")) - item = item.replace("
    ", "") - item = item.replace("
    ", "") - item = item.replace("
    ", " ") - item = item.replace("
    ", "") - item = item.replace("
    ", " ") - item = item.replace("
    ", "") - item = "\n".join(line for line in item.split("\n") if not line.isspace()) - return item - - def get_tests(self) -> list: - tests = [] - for item in self.items: - tests.extend(item.get_tests()) - return tests - - # Backward compatibility gather_tests = parse_docstring_to_DocumentationEntry_items diff --git a/mathics/doc/doc_entries.py b/mathics/doc/doc_entries.py new file mode 100644 index 000000000..1d423d1dd --- /dev/null +++ b/mathics/doc/doc_entries.py @@ -0,0 +1,548 @@ +""" +Documentation entries and doctests + +This module contains the objects representing the entries in the documentation +system, and the functions used to parse docstrings into these objects. + + +""" + +import logging +import re +from os import getenv +from typing import Callable, List, Optional + +from mathics.core.evaluation import Message, Print + +# Used for getting test results by test expresson and chapter/section information. +test_result_map = {} + + +# These are all the XML/HTML-like tags that documentation supports. +ALLOWED_TAGS = ( + "dl", + "dd", + "dt", + "em", + "url", + "ul", + "i", + "ol", + "li", + "con", + "console", + "img", + "imgpng", + "ref", + "subsection", +) +ALLOWED_TAGS_RE = dict( + (allowed, re.compile("<(%s.*?)>" % allowed)) for allowed in ALLOWED_TAGS +) + +# This string is used, so we can indicate a trailing blank at the end of a line by +# adding this string to the end of the line which gets stripped off. +# Some editors and formatters like to strip off trailing blanks at the ends of lines. +END_LINE_SENTINAL = "#<--#" + +# The regular expressions below (strings ending with _RE +# pull out information from docstring or text in a file. Ghetto parsing. + +CONSOLE_RE = re.compile(r"(?s)<(?Pcon|console)>(?P.*?)") +DL_ITEM_RE = re.compile( + r"(?s)<(?Pd[td])>(?P.*?)(?:|)\s*(?:(?=)|$)" +) +DL_RE = re.compile(r"(?s)
    (.*?)
    ") +HYPERTEXT_RE = re.compile( + r"(?s)<(?Pem|url)>(\s*:(?P.*?):\s*)?(?P.*?)" +) +IMG_PNG_RE = re.compile( + r'' +) +IMG_RE = re.compile( + r'' +) +# Preserve space before and after in-line code variables. +LATEX_RE = re.compile(r"(\s?)\$(\w+?)\$(\s?)") + +LIST_ITEM_RE = re.compile(r"(?s)
  • (.*?)(?:
    • |(?=
  • )|$)") +LIST_RE = re.compile(r"(?s)<(?Pul|ol)>(?P.*?)") +MATHICS_RE = re.compile(r"(?(.*?)") +QUOTATIONS_RE = re.compile(r"\"([\w\s,]*?)\"") +REF_RE = re.compile(r'') +SPECIAL_COMMANDS = { + "LaTeX": (r"LaTeX", r"\LaTeX{}"), + "Mathematica": ( + r"Mathematica®", + r"\emph{Mathematica}\textregistered{}", + ), + "Mathics": (r"Mathics3", r"\emph{Mathics3}"), + "Mathics3": (r"Mathics3", r"\emph{Mathics3}"), + "Sage": (r"Sage", r"\emph{Sage}"), + "Wolfram": (r"Wolfram", r"\emph{Wolfram}"), + "skip": (r"

    ", r"\bigskip"), +} +SUBSECTION_END_RE = re.compile("") + + +TESTCASE_RE = re.compile( + r"""(?mx)^ # re.MULTILINE (multi-line match) + # and re.VERBOSE (readable regular expressions + ((?:.|\n)*?) + ^\s+([>#SX])>[ ](.*) # test-code indicator + ((?:\n\s*(?:[:|=.][ ]|\.).*)*) # test-code results""" +) +TESTCASE_OUT_RE = re.compile(r"^\s*([:|=])(.*)$") + + +def get_results_by_test(test_expr: str, full_test_key: list, doc_data: dict) -> dict: + """ + Sometimes test numbering is off, either due to bugs or changes since the + data was read. + + Here, we compensate for this by looking up the test by its chapter and section name + portion stored in `full_test_key` along with the and the test expression data + stored in `test_expr`. + + This new key is looked up in `test_result_map` its value is returned. + + `doc_data` is only first time this is called to populate `test_result_map`. + """ + + # Strip off the test index form new key with this and the test string. + # Add to any existing value for that "result". This is now what we want to + # use as a tee in test_result_map to look for. + test_section = list(full_test_key)[:-1] + search_key = tuple(test_section) + + if not test_result_map: + # Populate test_result_map from doc_data + for key, result in doc_data.items(): + test_section = list(key)[:-1] + new_test_key = tuple(test_section) + next_result = test_result_map.get(new_test_key, None) + if next_result is None: + next_result = [result] + else: + next_result.append(result) + + test_result_map[new_test_key] = next_result + + results = test_result_map.get(search_key, None) + result = {} + if results: + for result_candidate in results: + if result_candidate["query"] == test_expr: + if result: + # Already found something + print(f"Warning, multiple results appear under {search_key}.") + return {} + + result = result_candidate + + return result + + +def filter_comments(doc: str) -> str: + """Remove docstring documentation comments. These are lines + that start with ##""" + return "\n".join( + line for line in doc.splitlines() if not line.lstrip().startswith("##") + ) + + +POST_SUBSTITUTION_TAG = "_POST_SUBSTITUTION%d_" + + +def pre_sub(regexp, text: str, repl_func): + """apply substitions previous to parse the text""" + post_substitutions = [] + + def repl_pre(match): + repl = repl_func(match) + index = len(post_substitutions) + post_substitutions.append(repl) + return POST_SUBSTITUTION_TAG % index + + text = regexp.sub(repl_pre, text) + + return text, post_substitutions + + +def post_sub(text: str, post_substitutions) -> str: + """apply substitions after parsing the doctests.""" + for index, sub in enumerate(post_substitutions): + text = text.replace(POST_SUBSTITUTION_TAG % index, sub) + return text + + +def parse_docstring_to_DocumentationEntry_items( + doc: str, + test_collection_constructor: Callable, + test_case_constructor: Callable, + text_constructor: Callable, + key_part=None, +) -> list: + """ + This parses string `doc` (using regular expressions) into Python objects. + The function returns a list of ``DocText`` and ``DocTests`` objects which + are contained in a ``DocumentationElement``. + + test_collection_constructor() is the class constructor call to create an + object for the test collection. + Each test is created via test_case_constructor(). + Text within the test is stored via text_constructor. + """ + # This function is used to populate a ``DocumentEntry`` element, that + # in principle is not associated to any container + # (``DocChapter``/``DocSection``/``DocSubsection``) + # of the documentation system. + # + # The ``key_part`` parameter was used to set the ``key`` of the + # ``DocTest`` elements. This attribute + # should be set just after the ``DocumentationEntry`` ( + # to which the tests belongs) is associated + # to a container, by calling ``container.set_parent_path``. + # However, the parameter is still used in MathicsDjango, so let's + # keep it and discard its value. + # + if key_part: + logging.warning("``key_part`` is deprecated. Its value is discarded.") + + # Remove commented lines. + doc = filter_comments(doc).strip(r"\s") + + # Remove leading
    ...
    + # doc = DL_RE.sub("", doc) + + # pre-substitute Python code because it might contain tests + doc, post_substitutions = pre_sub( + PYTHON_RE, doc, lambda m: "%s" % m.group(1) + ) + + # HACK: Artificially construct a last testcase to get the "intertext" + # after the last (real) testcase. Ignore the test, of course. + doc += "\n >> test\n = test" + testcases = TESTCASE_RE.findall(doc) + + tests = None + items = [] + for index, test_case in enumerate(testcases): + testcase = list(test_case) + text = testcase.pop(0).strip() + if text: + if tests is not None: + items.append(tests) + tests = None + text = post_sub(text, post_substitutions) + items.append(text_constructor(text)) + tests = None + if index < len(testcases) - 1: + test = test_case_constructor(index, testcase, None) + if tests is None: + tests = test_collection_constructor() + tests.tests.append(test) + + # If the last block in the loop was not a Text block, append the + # last set of tests. + if tests is not None: + items.append(tests) + tests = None + return items + + +class DocTest: + """ + Class to hold a single doctest. + + DocTest formatting rules: + + * `>>` Marks test case; it will also appear as part of + the documentation. + * `#>` Marks test private or one that does not appear as part of + the documentation. + * `X>` Shows the example in the docs, but disables testing the example. + * `S>` Shows the example in the docs, but disables testing if environment + variable SANDBOX is set. + * `=` Compares the result text. + * `:` Compares an (error) message. + `|` Prints output. + """ + + def __init__( + self, + index: int, + testcase: List[str], + key_prefix: Optional[tuple] = None, + ): + def strip_sentinal(line: str): + """Remove END_LINE_SENTINAL from the end of a line if it appears. + + Some editors like to strip blanks at the end of a line. + Since the line ends in END_LINE_SENTINAL which isn't blank, + any blanks that appear before will be preserved. + + Some tests require some lines to be blank or entry because + Mathics3 output can be that way + """ + if line.endswith(END_LINE_SENTINAL): + line = line[: -len(END_LINE_SENTINAL)] + + # Also remove any remaining trailing blanks since that + # seems *also* what we want to do. + return line.strip() + + self.index = index + self.outs = [] + self.result = None + + # Private test cases are executed, but NOT shown as part of the docs + self.private = testcase[0] == "#" + + # Ignored test cases are NOT executed, but shown as part of the docs + # Sandboxed test cases are NOT executed if environment SANDBOX is set + if testcase[0] == "X" or (testcase[0] == "S" and getenv("SANDBOX", False)): + self.ignore = True + # substitute '>' again so we get the correct formatting + testcase[0] = ">" + else: + self.ignore = False + + self.test = strip_sentinal(testcase[1]) + self._key = key_prefix + (index,) if key_prefix else None + + outs = testcase[2].splitlines() + for line in outs: + line = strip_sentinal(line) + if line: + if line.startswith("."): + text = line[1:] + if text.startswith(" "): + text = text[1:] + text = "\n" + text + if self.result is not None: + self.result += text + elif self.outs: + self.outs[-1].text += text + continue + + match = TESTCASE_OUT_RE.match(line) + if not match: + continue + symbol, text = match.group(1), match.group(2) + text = text.strip() + if symbol == "=": + self.result = text + elif symbol == ":": + out = Message("", "", text) + self.outs.append(out) + elif symbol == "|": + out = Print(text) + self.outs.append(out) + + def __str__(self) -> str: + return self.test + + @property + def key(self): + return self._key if hasattr(self, "_key") else None + + @key.setter + def key(self, value): + assert self.key is None + self._key = value + return self._key + + +class DocTests: + """ + A bunch of consecutive ``DocTest`` objects extracted from a Builtin docstring. + """ + + def __init__(self): + self.tests = [] + self.text = "" + + def get_tests(self) -> list: + """ + Returns lists test objects. + """ + return self.tests + + def is_private(self) -> bool: + return all(test.private for test in self.tests) + + def __str__(self) -> str: + return "\n".join(str(test) for test in self.tests) + + def test_indices(self) -> List[int]: + return [test.index for test in self.tests] + + +class DocText: + """ + Class to hold some (non-test) text. + + Some of the kinds of tags you may find here are showin in global ALLOWED_TAGS. + Some text may be marked with surrounding "$" or "'". + + The code here however does not make use of any of the tagging. + + """ + + def __init__(self, text): + self.text = text + + def __str__(self) -> str: + return self.text + + def get_tests(self) -> list: + """ + Return tests in a DocText item - there never are any. + """ + return [] + + def is_private(self) -> bool: + return False + + def test_indices(self) -> List[int]: + return [] + + +# Former XMLDoc +class DocumentationEntry: + """ + A class to hold the content of a documentation entry, + in our internal markdown-like format data. + + Describes the contain of an entry in the documentation system, as a + sequence (list) of items of the clase `DocText` and `DocTests`. + ``DocText`` items contains an internal XML-like formatted text. ``DocTests`` entries + contain one or more `DocTest` element. + Each level of the Documentation hierarchy contains an XMLDoc, describing the + content after the title and before the elements of the next level. For example, + in ``DocChapter``, ``DocChapter.doc`` contains the text coming after the title + of the chapter, and before the sections in `DocChapter.sections`. + Specialized classes like LaTeXDoc or and DjangoDoc provide methods for + getting formatted output. For LaTeXDoc ``latex()`` is added while for + DjangoDoc ``html()`` is added + Mathics core also uses this in getting usage strings (`??`). + + """ + + def __init__( + self, doc_str: str, title: str, section: Optional["DocSection"] = None + ): + self._set_classes() + self.title = title + self.path = None + if section: + chapter = section.chapter + part = chapter.part + # Note: we elide section.title + key_prefix = (part.title, chapter.title, title) + else: + key_prefix = None + + self.key_prefix = key_prefix + self.rawdoc = doc_str + self.items = parse_docstring_to_DocumentationEntry_items( + self.rawdoc, + self.docTest_collection_class, + self.docTest_class, + self.docText_class, + None, + ) + + def _set_classes(self): + """ + Tells to the initializator the classes to be used to build the items. + This must be overloaded by the daughter classes. + """ + if not hasattr(self, "docTest_collection_class"): + self.docTest_collection_class = DocTests + self.docTest_class = DocTest + self.docText_class = DocText + + def __str__(self) -> str: + return "\n\n".join(str(item) for item in self.items) + + def text(self) -> str: + # used for introspection + # TODO parse XML and pretty print + # HACK + item = str(self.items[0]) + item = "\n".join(line.strip() for line in item.split("\n")) + item = item.replace("
    ", "") + item = item.replace("
    ", "") + item = item.replace("
    ", " ") + item = item.replace("
    ", "") + item = item.replace("
    ", " ") + item = item.replace("
    ", "") + item = "\n".join(line for line in item.split("\n") if not line.isspace()) + return item + + def get_tests(self) -> list: + tests = [] + for item in self.items: + tests.extend(item.get_tests()) + return tests + + def set_parent_path(self, parent): + """Set the parent path""" + self.path = None + path = [] + while hasattr(parent, "parent"): + path = [parent.title] + path + parent = parent.parent + + if hasattr(parent, "title"): + path = [parent.title] + path + + if path: + self.path = path + # Set the key on each test + for test in self.get_tests(): + assert test.key is None + # For backward compatibility, we need + # to reduce this to three fields. + # TODO: remove me and ensure that this + # works here and in Mathics Django + if len(path) > 3: + path = path[:2] + [path[-1]] + test.key = tuple(path) + (test.index,) + + return self + + +class Tests: + """ + A group of tests in the same section or subsection. + """ + + def __init__( + self, + part_name: str, + chapter_name: str, + section_name: str, + doctests: List[DocTest], + subsection_name: Optional[str] = None, + ): + self.part = part_name + self.chapter = chapter_name + self.section = section_name + self.subsection = subsection_name + self.tests = doctests + self._key = None + + @property + def key(self): + return self._key + + @key.setter + def key(self, value): + assert self._key is None + self._key = value + return self._key diff --git a/mathics/doc/latex_doc.py b/mathics/doc/latex_doc.py index 2a9a5b343..34cb977d9 100644 --- a/mathics/doc/latex_doc.py +++ b/mathics/doc/latex_doc.py @@ -4,15 +4,23 @@ """ import re -from os import getenv from typing import Optional -from mathics.core.evaluation import Message, Print from mathics.doc.common_doc import ( + SUBSECTION_RE, + DocChapter, + DocGuideSection, + DocPart, + DocSection, + DocSubsection, + Documentation, + MathicsMainDocumentation, + sorted_chapters, +) +from mathics.doc.doc_entries import ( CONSOLE_RE, DL_ITEM_RE, DL_RE, - END_LINE_SENTINAL, HYPERTEXT_RE, IMG_PNG_RE, IMG_RE, @@ -25,26 +33,14 @@ REF_RE, SPECIAL_COMMANDS, SUBSECTION_END_RE, - SUBSECTION_RE, - TESTCASE_OUT_RE, - DocChapter, - DocGuideSection, - DocPart, - DocSection, - DocSubsection, DocTest, DocTests, DocText, - Documentation, DocumentationEntry, - MathicsMainDocumentation, get_results_by_test, - parse_docstring_to_DocumentationEntry_items, post_sub, pre_sub, - sorted_chapters, ) -from mathics.doc.utils import slugify # We keep track of the number of \begin{asy}'s we see so that # we can assocation asymptote file numbers with where they are @@ -481,72 +477,7 @@ class LaTeXDocTest(DocTest): """ def __init__(self, index, testcase, key_prefix=None): - def strip_sentinal(line): - """Remove END_LINE_SENTINAL from the end of a line if it appears. - - Some editors like to strip blanks at the end of a line. - Since the line ends in END_LINE_SENTINAL which isn't blank, - any blanks that appear before will be preserved. - - Some tests require some lines to be blank or entry because - Mathics output can be that way - """ - if line.endswith(END_LINE_SENTINAL): - line = line[: -len(END_LINE_SENTINAL)] - - # Also remove any remaining trailing blanks since that - # seems *also* what we want to do. - return line.strip() - - self.index = index - self.result = None - self.outs = [] - - # Private test cases are executed, but NOT shown as part of the docs - self.private = testcase[0] == "#" - - # Ignored test cases are NOT executed, but shown as part of the docs - # Sandboxed test cases are NOT executed if environment SANDBOX is set - if testcase[0] == "X" or (testcase[0] == "S" and getenv("SANDBOX", False)): - self.ignore = True - # substitute '>' again so we get the correct formatting - testcase[0] = ">" - else: - self.ignore = False - - self.test = strip_sentinal(testcase[1]) - - self.key = None - if key_prefix: - self.key = tuple(key_prefix + (index,)) - outs = testcase[2].splitlines() - for line in outs: - line = strip_sentinal(line) - if line: - if line.startswith("."): - text = line[1:] - if text.startswith(" "): - text = text[1:] - text = "\n" + text - if self.result is not None: - self.result += text - elif self.outs: - self.outs[-1].text += text - continue - - match = TESTCASE_OUT_RE.match(line) - if not match: - continue - symbol, text = match.group(1), match.group(2) - text = text.strip() - if symbol == "=": - self.result = text - elif symbol == ":": - out = Message("", "", text) - self.outs.append(out) - elif symbol == "|": - out = Print(text) - self.outs.append(out) + super().__init__(index, testcase, key_prefix) def __str__(self): return self.test @@ -769,27 +700,9 @@ def __init__( in_guide=False, summary_text="", ): - self.chapter = chapter - self.in_guide = in_guide - self.installed = installed - self.operator = operator - self.slug = slugify(title) - self.subsections = [] - self.subsections_by_slug = {} - self.summary_text = summary_text - self.title = title - - if text.count("
    ") != text.count("
    "): - raise ValueError( - "Missing opening or closing
    tag in " - "{} documentation".format(title) - ) - - # Needs to come after self.chapter is initialized since - # DocumentationEntry uses self.chapter. - self.doc = LaTeXDocumentationEntry(text, title, self) - - chapter.sections_by_slug[self.slug] = self + super().__init__( + chapter, title, text, operator, installed, in_guide, summary_text + ) def latex(self, doc_data: dict, quiet=False) -> str: """Render this Section object as LaTeX string and return that. @@ -839,7 +752,6 @@ def __init__( installed: bool = True, ): super().__init__(chapter, title, text, submodule, installed) - self.doc = LaTeXDocumentationEntry(text, title, self) def get_tests(self): # FIXME: The below is a little weird for Guide Sections. @@ -916,23 +828,6 @@ def __init__( super().__init__( chapter, section, title, text, operator, installed, in_guide, summary_text ) - self.doc = LaTeXDocumentationEntry(text, title, section) - - if in_guide: - # Tests haven't been picked out yet from the doc string yet. - # Gather them here. - self.items = parse_docstring_to_DocumentationEntry_items( - text, LaTeXDocTests, LaTeXDocTest, LaTeXDocText - ) - else: - self.items = [] - - if text.count("
    ") != text.count("
    "): - raise ValueError( - "Missing opening or closing
    tag in " - "{} documentation".format(title) - ) - self.section.subsections_by_slug[self.slug] = self def latex(self, doc_data: dict, quiet=False, chapters=None) -> str: """Render this Subsection object as LaTeX string and return that. diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py old mode 100755 new mode 100644 index 54fab4034..f8d5aa873 --- a/mathics/docpipeline.py +++ b/mathics/docpipeline.py @@ -25,13 +25,8 @@ from mathics.core.evaluation import Evaluation, Output from mathics.core.load_builtin import _builtins, import_and_load_builtins from mathics.core.parser import MathicsSingleLineFeeder -from mathics.doc.common_doc import ( - DocGuideSection, - DocSection, - DocTest, - DocTests, - MathicsMainDocumentation, -) +from mathics.doc.common_doc import DocGuideSection, DocSection, MathicsMainDocumentation +from mathics.doc.doc_entries import DocTest, DocTests from mathics.doc.utils import load_doctest_data, print_and_log from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule from mathics.timing import show_lru_cache_statistics @@ -53,7 +48,6 @@ def max_stored_size(self, _): CHECK_PARTIAL_ELAPSED_TIME = False LOGFILE = None - MAX_TESTS = 100000 # A number greater than the total number of tests. @@ -67,7 +61,6 @@ def doctest_compare(result: Optional[str], wanted: Optional[str]) -> bool: if result is None or wanted is None: return False - result_list = result.splitlines() wanted_list = wanted.splitlines() if result_list == [] and wanted_list == ["#<--#"]: @@ -102,7 +95,6 @@ def test_case( The test results are assumed to be foramtted to ASCII text. """ - global CHECK_PARTIAL_ELAPSED_TIME test_str, wanted_out, wanted = test.test, test.outs, test.result @@ -153,7 +145,6 @@ def fail(why): if CHECK_PARTIAL_ELAPSED_TIME: print(" comparison took ", datetime.now() - time_comparing) - if not comparison_result: print("result != wanted") fail_msg = f"Result: {result}\nWanted: {wanted}" @@ -186,11 +177,16 @@ def fail(why): def create_output(tests, doctest_data, output_format="latex"): + """ + Populate ``doctest_data`` with the results of the + ``tests`` in the format ``output_format`` + """ if DEFINITIONS is None: print_and_log(LOGFILE, "Definitions are not initialized.") return DEFINITIONS.reset_user_definitions() + for test in tests: if test.private: continue @@ -215,6 +211,37 @@ def create_output(tests, doctest_data, output_format="latex"): } +def load_pymathics_modules(module_names: set): + """ + Load pymathics modules + + PARAMETERS + ========== + + module_names: set + a set of modules to be loaded. + + Return + ====== + loaded_modules : set + the set of successfully loaded modules. + """ + loaded_modules = [] + for module_name in module_names: + try: + eval_LoadModule(module_name, DEFINITIONS) + except PyMathicsLoadException: + print(f"Python module {module_name} is not a Mathics3 module.") + + except Exception as e: + print(f"Python import errors with: {e}.") + else: + print(f"Mathics3 Module {module_name} loaded") + loaded_modules.append(module_name) + + return set(loaded_modules) + + def show_test_summary( total: int, failed: int, @@ -252,6 +279,10 @@ def show_test_summary( return +# +# TODO: Split and simplify this section +# +# def test_section_in_chapter( section: Union[DocSection, DocGuideSection], total: int, @@ -289,121 +320,65 @@ def test_section_in_chapter( part_name = chapter.part.title index = 0 if len(section.subsections) > 0: - for subsection in section.subsections: - if ( - include_subsections is not None - and subsection.title not in include_subsections - ): - continue + subsections = section.subsections + else: + subsections = [section] - DEFINITIONS.reset_user_definitions() - for test in subsection.get_tests(): - # Get key dropping off test index number - key = list(test.key)[1:-1] - if prev_key != key: - prev_key = key - section_name_for_print = " / ".join(key) - if quiet: - # We don't print with stars inside in test_case(), so print here. - print(f"Testing section: {section_name_for_print}") - index = 0 - else: - # Null out section name, so that on the next iteration we do not print a section header - # in test_case(). - section_name_for_print = "" - - if isinstance(test, DocTests): - for doctest in test.tests: - index += 1 - total += 1 - if not test_case( - doctest, - total, - index, - quiet=quiet, - section_name=section_name, - section_for_print=section_name_for_print, - chapter_name=chapter_name, - part=part_name, - ): - failed += 1 - if stop_on_failure: - break - elif test.ignore: - continue + if chapter.doc: + subsections = [chapter.doc] + subsections - else: - index += 1 - - if index < start_at: - skipped += 1 - continue - - total += 1 - if not test_case( - test, - total, - index, - quiet=quiet, - section_name=section_name, - section_for_print=section_name_for_print, - chapter_name=chapter_name, - part=part_name, - ): - failed += 1 - if stop_on_failure: - break - pass - pass - pass - pass - else: - if include_subsections is None or section.title in include_subsections: - DEFINITIONS.reset_user_definitions() - for test in section.get_tests(): - # Get key dropping off test index number - key = list(test.key)[1:-1] - if prev_key != key: - prev_key = key - section_name_for_print = " / ".join(key) - if quiet: - print(f"Testing section: {section_name_for_print}") - index = 0 - else: - # Null out section name, so that on the next iteration we do not print a section header. - section_name_for_print = "" + for subsection in subsections: + if ( + include_subsections is not None + and subsection.title not in include_subsections + ): + continue - if test.ignore: - continue + DEFINITIONS.reset_user_definitions() + for test in subsection.get_tests(): + # Get key dropping off test index number + key = list(test.key)[1:-1] + if prev_key != key: + prev_key = key + section_name_for_print = " / ".join(key) + if quiet: + # We don't print with stars inside in test_case(), so print here. + print(f"Testing section: {section_name_for_print}") + index = 0 + else: + # Null out section name, so that on the next iteration we do not print a section header + # in test_case(). + section_name_for_print = "" - else: - index += 1 + tests = test.tests if isinstance(test, DocTests) else [test] - if index < start_at: - skipped += 1 - continue + for doctest in tests: + if doctest.ignore: + continue - total += 1 - if total >= max_tests: - break + index += 1 + total += 1 + if index < start_at: + skipped += 1 + continue - if not test_case( - test, - total, - index, - quiet=quiet, - section_name=section.title, - section_for_print=section_name_for_print, - chapter_name=chapter.title, - part=part_name, - ): - failed += 1 - if stop_on_failure: - break - pass - pass + if not test_case( + doctest, + total, + index, + quiet=quiet, + section_name=section_name, + section_for_print=section_name_for_print, + chapter_name=chapter_name, + part=part_name, + ): + failed += 1 + if stop_on_failure: + break + # If failed, do not continue with the other subsections. + if failed and stop_on_failure: + break - pass return total, failed, prev_key @@ -475,10 +450,15 @@ def test_tests( """ - total = index = failed = skipped = 0 + total = failed = skipped = 0 prev_key = [] failed_symbols = set() + # For consistency set the character encoding ASCII which is + # the lowest common denominator available on all systems. + settings.SYSTEM_CHARACTER_ENCODING = "ASCII" + DEFINITIONS.reset_user_definitions() + output_data, names = validate_group_setup( set(), None, @@ -510,10 +490,15 @@ def test_tests( start_at=start_at, max_tests=max_tests, ) - if generate_output and failed == 0: - create_output(section.doc.get_tests(), output_data) - pass - pass + if failed and stop_on_failure: + break + else: + if generate_output: + create_output(section.doc.get_tests(), output_data) + if failed and stop_on_failure: + break + if failed and stop_on_failure: + break show_test_summary( total, @@ -527,6 +512,7 @@ def test_tests( if generate_output and (failed == 0 or keep_going): save_doctest_data(output_data) + return total, failed, skipped, failed_symbols, index @@ -585,12 +571,10 @@ def test_chapters( create_output(section.doc.get_tests(), output_data) pass pass - + # Shortcut: if we already pass through all the + # include_chapters, break the loop if seen_chapters == include_chapters: break - if chapter_name in include_chapters: - seen_chapters.add(chapter_name) - pass show_test_summary( total, @@ -666,7 +650,6 @@ def test_sections( seen_sections.add(section_name_for_finish) last_section_name = section_name_for_finish pass - if seen_last_section: break pass @@ -948,16 +931,8 @@ def main(): # LoadModule Mathics3 modules if args.pymathics: - for module_name in args.pymathics.split(","): - try: - eval_LoadModule(module_name, DEFINITIONS) - except PyMathicsLoadException: - print(f"Python module {module_name} is not a Mathics3 module.") - - except Exception as e: - print(f"Python import errors with: {e}.") - else: - print(f"Mathics3 Module {module_name} loaded") + required_modules = set(args.pymathics.split(",")) + load_pymathics_modules(required_modules) DOCUMENTATION.load_documentation_sources() diff --git a/test/doc/test_common.py b/test/doc/test_common.py index d8dd5b19f..8d7ff17e7 100644 --- a/test/doc/test_common.py +++ b/test/doc/test_common.py @@ -9,12 +9,14 @@ DocChapter, DocPart, DocSection, + Documentation, + MathicsMainDocumentation, +) +from mathics.doc.doc_entries import ( DocTest, DocTests, DocText, - Documentation, DocumentationEntry, - MathicsMainDocumentation, parse_docstring_to_DocumentationEntry_items, ) from mathics.settings import DOC_DIR diff --git a/test/doc/test_latex.py b/test/doc/test_latex.py index ddc37bae5..4e4e9a1cc 100644 --- a/test/doc/test_latex.py +++ b/test/doc/test_latex.py @@ -5,6 +5,7 @@ from mathics.core.evaluation import Message, Print from mathics.core.load_builtin import import_and_load_builtins +from mathics.doc.doc_entries import parse_docstring_to_DocumentationEntry_items from mathics.doc.latex_doc import ( LaTeXDocChapter, LaTeXDocPart, @@ -14,7 +15,6 @@ LaTeXDocText, LaTeXDocumentationEntry, LaTeXMathicsDocumentation, - parse_docstring_to_DocumentationEntry_items, ) from mathics.settings import DOC_DIR From 2f6eabbb6558d74ea0238014760168c28746d811 Mon Sep 17 00:00:00 2001 From: mmatera Date: Sun, 24 Mar 2024 23:20:33 -0300 Subject: [PATCH 2/2] black --- mathics/docpipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py index c952ac5e5..887083a12 100644 --- a/mathics/docpipeline.py +++ b/mathics/docpipeline.py @@ -519,6 +519,7 @@ def show_and_return(): return show_and_return() + def test_chapters( include_chapters: set, quiet=False,