Merge pull request #113 from matze-dd/Issue-109

Issue 109

HISTORY.md

@@ -1,17 +1,18 @@
 Work in progress
 ----------------
-- **Changed interface** to extension modules for packages and document classes
-  (issue [#110](../../issues/110)).
-  Entry point is now function init\_module() with two arguments: parser and
-  list of packet options.
-  See for instance file yalafi/packages/amsmath.py.
 - yalafi.shell
   - added support for multi-language documents (issue [#98](../../issues/98))
   - new options --multi-language, --ml-continue-threshold, --ml-rule-threshold,
     --ml-disable, --ml-disablecategories
 - yalafi core
+  - **Changed interface** to extension modules for packages and document
+    classes (issue [#110](../../issues/110)).
+    Entry point is now function init\_module() with two arguments: parser and
+    list of package options.
+    See for instance yalafi/packages/amsmath.py and yalafi/packages/babel.py.
   - added support for multi-language documents (issue [#98](../../issues/98));
-    including fixed issues [#104](../../issues/104), [#108](../../issues/108)
+    including fixed issues [#104](../../issues/104), [#108](../../issues/108),
+    [#109](../../issues/109)
 - README.md: updated, added section for multi-language support
 
 Version 1.1.7 (2020/11/04)

README.md

@@ -1183,12 +1183,11 @@ This feature is experimental, any comments are welcome.
 Operation may be slow, unless a LanguageTool server is used, for instance,
 via option '--server my'.
 
-As an example, assume option '--multi-language' for yalafi.shell and input:
+As an example, assume option '--multi-language' for yalafi.shell and the LaTeX
+text:
 ```
 \documentclass{article}
 \usepackage[german,english]{babel}
-
-\selectlanguage{english}
 \newcommand{\german}[1]{\textit{\foreignlanguage{german}{#1}}}
 
 \begin{document}
@@ -1199,17 +1198,16 @@ Then, the Vim example from section [“Plain Vim”](#plain-vim)
 with setting `let g:ltyc_showsuggestions = 1` will produce this quickfix
 window:
 ```
-t.tex|8 col 9 info|  Possible spelling mistake found. Suggestion: the; then; they; them; thee; Theo; hex; THX; TeX; Tex; The; t hex; the x; Théo
-t.tex|8 col 34 info|  Möglicher Tippfehler gefunden. Suggestion: exzellent; exzellente; exzellenten; exzellenter; Exzellenz; exzellentes; erzählend; exzellentem; erhellend; erkältend; exzelliert
-t.tex|8 col 44 info|  Two consecutive dots Suggestion: .; …
+t.tex|6 col 9 info|  Possible spelling mistake found. Suggestion: the; then; they; them; thee; Theo; hex; THX; TeX; Tex; The; t hex; the x; Théo
+t.tex|6 col 34 info|  Möglicher Tippfehler gefunden. Suggestion: exzellent; exzellente; exzellenten; exzellenter; Exzellenz; exzellentes; erzählend; exzellentem; erhellend; erkältend; exzelliert
+t.tex|6 col 44 info|  Two consecutive dots Suggestion: .; …
 ```
-The initial language is specified by option --language.
-Commands like `\selectlanguage{...}` are effective in files loaded via option
---define or with `\LTinput{...}`.
-Language names in 'babel' commands are mapped to xx-XX codes by dictionary
+The initial language is specified by option --language, it is overwritten
+upon `\usepackage[...]{babel}`.
+Commands like `\selectlanguage{...}` are also effective in files loaded via
+option --define or with `\LTinput{...}`.
+Language names in babel commands are mapped to xx-XX codes by dictionary
 'language\_map' in file [yalafi/packages/babel.py](yalafi/packages/babel.py).
-Please note:
-- Currently, `\usepackage[...]{babel}` does not set the active language.
 
 **Further options.**
 In the above example, LanguageTool is invoked for

tests/test_multi_lang.py

@@ -236,3 +236,31 @@ def test_10():
     plain = get_ml_txt(latex_10, lang='en-GB')
     assert plain == plain_10
 
+#   issue #109
+#
+latex_11 = r"""
+"a
+\usepackage{babel}
+"a
+\usepackage[english,german]{babel}
+"a
+\selectlanguage{english}
+"a
+"""
+plain_11 = {
+'de-DE': [
+"""ä
+"""
+],
+'en-GB': [
+"""
+"a
+"a
+""", """"a
+"""
+],
+}
+def test_11():
+    plain = get_ml_txt(latex_11, lang='en-GB')
+    assert plain == plain_11
+

tests/test_shell/test_shell.py

@@ -21,7 +21,7 @@ def test_shell():
     # start "LT server"
     subprocess.Popen(server_cmd.split(), stdout=subprocess.DEVNULL,
                         stderr=subprocess.DEVNULL)
-    time.sleep(10)
+    time.sleep(5)
 
     # run shell.py
     out = subprocess.run(shell_cmd.split(), stdout=subprocess.PIPE,

tests/test_shell_cmd/run_shell.py

@@ -96,7 +96,7 @@ def run_as_server(options, latex, json):
                     + ' ' + options)
     server = subprocess.Popen(cmd.split(), stdout=subprocess.DEVNULL,
                                     stderr=subprocess.DEVNULL)
-    time.sleep(10)
+    time.sleep(5)
 
     cmd = 'python -m yalafi.shell --plain-input --server my ' + shell_in
     out = subprocess.run(cmd.split(), stdout=subprocess.PIPE)

yalafi/defs.py

@@ -19,10 +19,12 @@
 from . import utils
 
 class InitModule:
-    def __init__(self, macros_latex='', macros_python=[], environments=[]):
+    def __init__(self, macros_latex='', macros_python=[], environments=[],
+                                        inject_tokens=[]):
         self.macros_latex = macros_latex
         self.macros_python = macros_python
         self.environs = environments
+        self.inject_tokens = inject_tokens
 
 class Printable:
     def __repr__(self):

yalafi/handlers.py

@@ -126,16 +126,17 @@ def h_load_defs(parser, buf, mac, args, pos):
         return utils.latex_error('could not read file ' + repr(file),
                                         pos, parser.latex, parser.parms)
     toks = parser.parser_work(latex)
-    return utils.filter_lang_toks(toks, pos)
+    return utils.filter_set_toks(toks, pos, defs.LanguageToken)
 
 #   read definitions for a LaTeX package
 #
 def h_load_module(prefix):
     def f(parser, buf, mac, args, pos):
-        options = []
+        options = parser.parse_keyvals_list(args[0])
+        options = parser.expand_keyvals(options)
         pack = parser.get_text_expanded(args[1])
         f = utils.get_module_handler(pack, prefix)
-        parser.init_package(pack, f, options)
-        return []
+        out = parser.init_package(pack, f, options)
+        return utils.filter_set_toks(out, pos, None)
     return f
 

yalafi/mathparser.py

@@ -72,7 +72,7 @@ def expand_display_math(self, buf, tok, env):
             tokens = self.detect_math_parts(tokens)
             sec, next_repl = self.replace_section(False, tokens,
                         first_section, next_repl,
-                        self.parser.parms.lang_context().math_repl_display)
+                        self.parser.parms.lang_context.math_repl_display)
             out += sec
             if end and end.txt == '&':
                 out.append(defs.SpaceToken(out[-1].pos, ' ', pos_fix=True))
@@ -98,7 +98,7 @@ def expand_display_math(self, buf, tok, env):
                 out = [defs.ActionToken(start_simple),
                         defs.SpaceToken(start_simple, '  ', pos_fix=True),
                         defs.TextToken(start_simple, self.parser.parms.
-                                        lang_context().math_repl_display[0],
+                                        lang_context.math_repl_display[0],
                                         pos_fix=True)]
                 if txt and txt[-1] in self.parser.parms.math_punctuation:
                     out.append(defs.TextToken(out[-1].pos, txt[-1],
@@ -114,7 +114,7 @@ def expand_inline_math(self, buf, tok):
         tokens = self.detect_math_parts(tokens)
         out = [defs.ActionToken(tok.pos)]
         t, x = self.replace_section(True, tokens, True, True,
-                            self.parser.parms.lang_context().math_repl_inline)
+                            self.parser.parms.lang_context.math_repl_inline)
         out += t
         out.append(defs.ActionToken(out[-1].pos))
         return out
@@ -241,8 +241,8 @@ def replace_section(self, inline, tokens, first_section,
             op = tok.leading_op()
             elem = tok.has_elem(parms)
             if not inline and first_part and op:
-                s = parms.lang_context().math_op_text.get(
-                            op.txt, parms.lang_context().math_op_text[None])
+                s = parms.lang_context.math_op_text.get(
+                            op.txt, parms.lang_context.math_op_text[None])
                 out.append(defs.SpaceToken(tok.pos, ' ', pos_fix=True))
                 out.append(defs.TextToken(op.pos, s, pos_fix=True))
                 out.append(defs.SpaceToken(op.pos, ' ', pos_fix=True))

yalafi/packages/amsthm.py

@@ -32,7 +32,7 @@ def h_proof(parser, buf, mac, args, pos):
     if args[0]:
         ret = args[0]
     else:
-        ret = [TextToken(pos, parser.parms.lang_context().proof_name,
+        ret = [TextToken(pos, parser.parms.lang_context.proof_name,
                         pos_fix=True)]
     return ret + [TextToken(ret[-1].pos, '.', pos_fix=True),
                         SpaceToken(ret[-1].pos, '\n', pos_fix=True)]

yalafi/packages/babel.py

@@ -72,8 +72,14 @@ def init_module(parser, options):
 
     environments = []
 
+    inject_tokens = []
+    if options:
+        # set current language to the last in option list
+        inject_tokens = [LanguageToken(0, lang=translate_lang(options[-1][0]),
+                                                hard=True)]
+
     return InitModule(macros_latex=macros_latex, macros_python=macros_python,
-                        environments=environments)
+                        environments=environments, inject_tokens=inject_tokens)
 
 def modify_language_map(babel, lt):
     language_map[babel] = lt

yalafi/packages/glossaries.py

@@ -118,7 +118,7 @@ def h_newacronym(parser, buf, mac, args, pos):
     return modify_description(parser, args[2])
 
 def h_newglossaryentry(parser, buf, mac, args, pos):
-    descr = parser.parse_keyval_list(args[1]).get('description', [])
+    descr = parser.parse_keyvals_dict(args[1]).get('description', [])
     return modify_description(parser, descr)
 
 the_glossary = {}
@@ -127,7 +127,7 @@ def h_newglossaryentry(parser, buf, mac, args, pos):
 #
 def h_parse_glsdefs(parser, buf, mac, args, pos):
     label = parser.get_text_expanded(args[0])
-    the_glossary[label] = parser.parse_keyval_list(args[1])
+    the_glossary[label] = parser.parse_keyvals_dict(args[1])
     return []
 
 #   get token list for a key, database label given as token list

yalafi/parameters.py

@@ -218,6 +218,7 @@ def init_parser_languages(self, lang):
 
         self.parser_lang_stack = [(settings[self.check_parser_lang(lang)],
                                         lang)]
+        self.lang_context = self.parser_lang_stack[-1][0]
 
     #   set misc collections
     #
@@ -433,9 +434,7 @@ def change_parser_lang(self, tok):
                 self.parser_lang_stack.append(
                 (self.parser_lang_settings[self.check_parser_lang(tok.lang)],
                         tok.lang))
-
-    def lang_context(self):
-        return self.parser_lang_stack[-1][0]
+        self.lang_context = self.parser_lang_stack[-1][0]
 
     def lang_context_lang(self):
         return self.parser_lang_stack[-1][1]

yalafi/parser.py

@@ -28,7 +28,7 @@ class Parser:
     def __init__(self, parms, packages=[], read_macros=None):
         self.parms = parms
         self.read_macros = read_macros
-        self.packages = []
+        self.packages = {}
         self.the_macros = {}
         self.the_environments = {}
         self.mathparser = mathparser.MathParser(self)
@@ -65,20 +65,24 @@ def labs_default(level):
     #   - load other required packages
     #   - let package handler update parameter object
     #   - update local dictionaries for macros and environments
+    #   --> only do all that if package not yet loaded, or if options have
+    #       changed
     #
     def init_package(self, name, actions, options):
-        if name and name in self.packages:
-            return
+        out = []
+        if name and name in self.packages and self.packages[name] == options:
+            return out
         try:
             for requ in actions[0]:
-                if requ not in self.packages:
-                    self.init_package(requ, utils.get_module_handler(
+                if requ not in self.packages or self.packages[requ] == options:
+                    out += self.init_package(requ, utils.get_module_handler(
                                     requ, self.parms.package_modules), options)
             if name:
-                self.packages.append(name)
-            self.modify_parameters(actions[1], options)
+                self.packages[name] = options
+            out += self.modify_parameters(actions[1], options)
         except:
             utils.fatal('error loading module ' + repr(name))
+        return out
 
     #   modify local parameter object
     #   - used by package extension mechanism
@@ -92,6 +96,7 @@ def modify_parameters(self, f, options):
             self.the_environments[e.name] = e
         if mods.macros_latex:
             self.parser_work(mods.macros_latex)
+        return mods.inject_tokens
 
     #   scan and parse (expand) LaTeX string to tokens
     #   - here also skip LaTeX text enclosed in special comments
@@ -141,7 +146,7 @@ def parse(self, latex, define='', extract=None):
         main = []
         if define:
             toks = self.parser_work(define)
-            main = utils.filter_lang_toks(toks, 0)
+            main = utils.filter_set_toks(toks, 0, defs.LanguageToken)
         main += self.parser_work(latex)
 
         if extract:
@@ -249,7 +254,7 @@ def expand_sequence(self, buf, env_stop=None):
                 if self.parms.multi_language:
                     self.parms.change_parser_lang(tok)
                     out.append(tok)
-            elif tok.txt in self.parms.lang_context().active_chars:
+            elif tok.txt in self.parms.lang_context.active_chars:
                 out.append(self.expand_short_macro(buf, tok))
                 continue
             elif type(tok) is defs.CommentToken:
@@ -588,21 +593,25 @@ def Space(pos):
     def expand_short_macro(self, buf, tok):
         cur = buf.next()
         if (not cur or tok.txt + cur.txt
-                    not in self.parms.lang_context().short_macros):
+                    not in self.parms.lang_context.short_macros):
             return tok
         buf.next()
         return defs.TextToken(tok.pos,
-                    self.parms.lang_context().short_macros[tok.txt + cur.txt],
+                    self.parms.lang_context.short_macros[tok.txt + cur.txt],
                     pos_fix=True)
 
     #   parse a key-value list of the form 'a=b,a4paper,x={ hoho }'
     #   - return a dictionary
     #   - values are token lists, except if no '=' given: value then is None
     #   XXX: parsing of key name is not really robust
     #
-    def parse_keyval_list(self, tokens):
+    def parse_keyvals_dict(self, tokens):
+        as_list = self.parse_keyvals_list(tokens)
+        return {k: v for (k, v) in as_list}
+
+    def parse_keyvals_list(self, tokens):
         buf = scanner.Buffer(tokens)
-        values = {}
+        values = []
         while True:
             tok = buf.skip_space()
             if not tok:
@@ -616,7 +625,7 @@ def parse_keyval_list(self, tokens):
             if not tok or tok.txt == ',':
                 # no value given with = ...
                 tok = buf.next()
-                values[key] = None
+                values.append((key, None))
                 continue
             buf.next()                  # skip '='
             tok = buf.skip_space()      # skip leading space of value
@@ -640,7 +649,16 @@ def parse_keyval_list(self, tokens):
             if val and type(val[-1]) is defs.SpaceToken:
                 # skip trailing space of value
                 val.pop()
-            values[key] = val
+            values.append((key, val))
             buf.next()                  # skip ','
         return values
 
+    #   expand keyval list to text form (None if no '=' given)
+    #
+    def expand_keyvals(self, keyvals):
+        def f(kv):
+            if kv[1] is None:
+                return kv
+            return (kv[0], self.get_text_expanded(kv[1]))
+        return [f(kv) for kv in keyvals]
+

yalafi/utils.py

@@ -127,14 +127,14 @@ def get_module_handler(name, prefix):
         warning('could not load module ' + repr(mod))
         return [], lambda p, o: defs.InitModule()
 
-#   filter out LanguageTokens and set their position numbers
+#   filter out tokens and set their position numbers
 #
-def filter_lang_toks(toks, pos):
+def filter_set_toks(toks, pos, tok_typ):
     def f(t):
         t = copy.copy(t)
         t.pos = pos
         return t
-    return [f(t) for t in toks if type(t) is defs.LanguageToken]
+    return [f(t) for t in toks if tok_typ is None or type(t) is tok_typ]
 
 
 class LanguageSection: