otk: fix various corner cases, add json support, fix externals

src/otk/annotation.py

@@ -1,3 +1,8 @@
+import json
+
+from json import JSONEncoder
+from typing import Any
+
 
 class OtkValueMixin:
     @property
@@ -13,22 +18,80 @@ class OtkDict(OtkValueMixin, dict):
     def __init__(self, other: dict):
         self.update(other)
 
+    # XXX: not a good nameq
+    def otk_dup(self, new: dict):
+        new = OtkDict(new)
+        new.otk_src = self.otk_src
+        return new
+
 
 class OtkList(OtkValueMixin, list):
     def __init__(self, other: list):
         self.extend(other)
-    def replace(self, new: list):
-        self.clear()
-        self.extend(new)
+
+    # XXX: not a good name
+    def otk_dup(self, new: list):
+        new = OtkList(new)
+        new.otk_src = self.otk_src
+        return new
 
 
 class OtkStr(OtkValueMixin, str):
-   def __new__(cls, other):
+    def __new__(cls, other):
         val = super().__new__(cls, other)
         return val
 
+    # XXX: not a good name
+    def otk_dup(self, new):
+        # str are immutable
+        new = OtkStr(new)
+        new.otk_src = self.otk_src
+        return new
+
 
 class OtkInt(OtkValueMixin, int):
-   def __new__(cls, other):
+    def __new__(cls, other):
         val = super().__new__(cls, other)
         return val
+
+
+# bool can't be subclassed so we need to workaround
+class OtkBool(OtkValueMixin):
+    def __init__(self, other) -> None:
+        self._bool = other
+
+
+# this is only needed to support OtkBool
+class OtkJSONEncoder(JSONEncoder):
+    def default(self, o):
+        if isinstance(o, OtkBool):
+            return o._bool
+
+
+# this is needed for the external data which does not come in via
+# the yaml loader
+def otk_deep_convert_from(origin: OtkValueMixin, data: Any):
+    ret = data
+    if isinstance(data, OtkValueMixin):
+        return ret
+    if isinstance(data, dict):
+        ret = OtkDict({
+            key: otk_deep_convert_from(origin, value)
+            for key, value in data.items()
+        })
+    elif isinstance(data, list):
+        ret = OtkList([
+            otk_deep_convert_from(origin, item) for item in data
+        ])
+    elif isinstance(data, str):
+        ret = OtkStr(data)
+    # must be before int, because "isinstance(True, int) == True"
+    elif isinstance(data, bool):
+        ret = OtkBool(data)
+    elif isinstance(data, int):
+        ret = OtkInt(data)
+
+    # None canot be subclasssed nor annotated :(
+    if not ret is None:
+        ret.otk_src = origin
+    return ret

src/otk/context.py

@@ -107,7 +107,7 @@ def variable(self, name: str) -> Any:
                 value = value[part]
             elif isinstance(value, list):
                 if not part.isnumeric():
-                    raise TransformVariableIndexTypeError(f"part is not numeric but {type(part)}")
+                    raise TransformVariableIndexTypeError(f"part {part} is not numeric but {type(part)} for {value}")
 
                 try:
                     value = value[int(part)]

src/otk/external.py

@@ -9,6 +9,7 @@
 import os
 from typing import Any
 
+from .annotation import OtkDict, otk_deep_convert_from, OtkJSONEncoder
 from .constant import PREFIX_EXTERNAL
 from .error import ExternalFailedError
 from .traversal import State
@@ -23,7 +24,7 @@ def call(state: State, directive: str, tree: Any) -> Any:
     data = json.dumps(
         {
             "tree": tree,
-        }
+        }, cls=OtkJSONEncoder
     )
 
     process = subprocess.run([exe], input=data, encoding="utf8", capture_output=True, check=False)
@@ -33,7 +34,9 @@ def call(state: State, directive: str, tree: Any) -> Any:
         raise ExternalFailedError(msg, state)
 
     res = json.loads(process.stdout)
-    return res["tree"]
+    # we may need a deep convert here if we want otk_src information
+    # on the childreen
+    return otk_deep_convert_from(tree, res["tree"])
 
 
 def exe_from_directive(directive):

src/otk/loader.py

@@ -0,0 +1,68 @@
+import yaml
+
+from .annotation import OtkDict, OtkList, OtkStr, OtkInt
+from .error import (
+    IncludeNotFoundError, OTKError,
+    ParseError, ParseTypeError, ParseValueError, ParseDuplicatedYamlKeyError,
+    TransformDirectiveTypeError, TransformDirectiveUnknownError,
+)
+
+
+# from https://gist.github.com/pypt/94d747fe5180851196eb?permalink_comment_id=4653474#gistcomment-4653474
+# pylint: disable=too-many-ancestors
+class SafeOtkLoader(yaml.SafeLoader):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, self.otk_dict_constructor)
+        self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, self.otk_list_constructor)
+        self.add_constructor('tag:yaml.org,2002:str', self.otk_construct_yaml_str)
+        self.add_constructor('tag:yaml.org,2002:int', self.otk_construct_yaml_int)
+
+    def otk_src_from(self, node):
+        line = node.start_mark.line
+        # XXX: why is this needed :(
+        if isinstance(node, yaml.nodes.ScalarNode):
+            line += 1
+        # XXX: make this a dataclass instead
+        return f"{node.start_mark.name}:{line}"
+
+    def construct_mapping(self, node, deep=False):
+        mapping = set()
+        # check duplicates
+        for key_node, _ in node.value:
+            if ':merge' in key_node.tag:
+                continue
+            key = self.construct_object(key_node, deep=deep)
+            if key in mapping:
+                if "otk." in key:
+                    raise ParseDuplicatedYamlKeyError(
+                        f"duplicated {key!r} key found, try using "
+                        f"{key}.<uniq-tag>, e.g. {key}.foo")
+                raise ParseDuplicatedYamlKeyError(f"duplicated {key!r} key found")
+            mapping.add(key)
+        return super().construct_mapping(node, deep)
+
+    def otk_dict_constructor(self, loader, node):
+        data = loader.construct_mapping(node)
+        otk_dict = OtkDict(data)
+        otk_dict.otk_src = self.otk_src_from(node)
+        return otk_dict
+
+    def otk_list_constructor(self, loader, node):
+        data = loader.construct_sequence(node)
+        otk_list = OtkList(data)
+        otk_list.otk_src = self.otk_src_from(node)
+        return otk_list
+
+    def otk_construct_yaml_str(self, loader, node):
+        data = loader.construct_scalar(node)
+        otk_scalar = OtkStr(data)
+        otk_scalar.otk_src = self.otk_src_from(node)
+        return otk_scalar
+
+    def otk_construct_yaml_int(self, loader, node):
+        data = super().construct_yaml_int(node)
+        otk_int = OtkInt(data)
+        otk_int.otk_src = self.otk_src_from(node)
+        return otk_int

src/otk/target.py

@@ -3,6 +3,7 @@
 from abc import ABC, abstractmethod
 from typing import Any
 
+from .annotation import OtkJSONEncoder
 from .context import CommonContext, OSBuildContext
 from .constant import PREFIX_TARGET
 from .error import ParseError
@@ -25,7 +26,8 @@ def ensure_valid(self, _tree: Any) -> None:
         pass
 
     def as_string(self, context: CommonContext, tree: Any, pretty: bool = True) -> str:
-        return json.dumps(tree, indent=2 if pretty else None)
+        return json.dumps(tree, indent=2 if pretty else None,
+                          cls=OtkJSONEncoder)
 
 
 class OSBuildTarget(Target):
@@ -39,4 +41,5 @@ def as_string(self, context: OSBuildContext, tree: Any, pretty: bool = True) ->
         osbuild_tree = tree[PREFIX_TARGET + context.target_requested]
         osbuild_tree["version"] = "2"
 
-        return json.dumps(osbuild_tree, indent=2 if pretty else None)
+        return json.dumps(osbuild_tree, indent=2 if pretty else None,
+                          cls=OtkJSONEncoder)

src/otk/transform.py

@@ -21,7 +21,7 @@
 import yaml
 
 from . import tree
-from .annotation import OtkDict, OtkList, OtkStr, OtkInt
+from .annotation import OtkBool, OtkDict, OtkList, OtkStr, OtkInt
 from .constant import NAME_VERSION, PREFIX, PREFIX_DEFINE, PREFIX_INCLUDE, PREFIX_OP, PREFIX_TARGET
 from .context import Context, validate_var_name
 from .error import (
@@ -30,84 +30,26 @@
     TransformDirectiveTypeError, TransformDirectiveUnknownError,
 )
 from .external import call
+from .loader import SafeOtkLoader
 from .traversal import State
 
 log = logging.getLogger(__name__)
 
 
-# from https://gist.github.com/pypt/94d747fe5180851196eb?permalink_comment_id=4653474#gistcomment-4653474
-# pylint: disable=too-many-ancestors
-class SafeOtkLoader(yaml.SafeLoader):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, self.otk_dict_constructor)
-        self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, self.otk_list_constructor)
-        self.add_constructor('tag:yaml.org,2002:str', self.otk_construct_yaml_str)
-        self.add_constructor('tag:yaml.org,2002:int', self.otk_construct_yaml_int)
-
-    def otk_src_from(self, node):
-        line = node.start_mark.line
-        # XXX: why is this needed :(
-        if isinstance(node, yaml.nodes.ScalarNode):
-            line += 1
-        return f"{node.start_mark.name}:{line}"
-
-    def construct_mapping(self, node, deep=False):
-        mapping = set()
-        # check duplicates
-        for key_node, _ in node.value:
-            if ':merge' in key_node.tag:
-                continue
-            key = self.construct_object(key_node, deep=deep)
-            if key in mapping:
-                if "otk." in key:
-                    raise ParseDuplicatedYamlKeyError(
-                        f"duplicated {key!r} key found, try using "
-                        f"{key}.<uniq-tag>, e.g. {key}.foo")
-                raise ParseDuplicatedYamlKeyError(f"duplicated {key!r} key found")
-            mapping.add(key)
-        return super().construct_mapping(node, deep)
-
-    def otk_dict_constructor(self, loader, node):
-        data = loader.construct_mapping(node)
-        otk_dict = OtkDict(data)
-        otk_dict.otk_src = self.otk_src_from(node)
-        return otk_dict
-
-    def otk_list_constructor(self, loader, node):
-        data = loader.construct_sequence(node)
-        otk_list = OtkList(data)
-        otk_list.otk_src = self.otk_src_from(node)
-        return otk_list
-
-    def otk_construct_yaml_str(self, loader, node):
-        data = loader.construct_scalar(node)
-        otk_scalar = OtkStr(data)
-        otk_scalar.otk_src = self.otk_src_from(node)
-        return otk_scalar
-
-    def otk_construct_yaml_int(self, loader, node):
-        data = super().construct_yaml_int(node)
-        otk_int = OtkInt(data)
-        otk_int.otk_src = self.otk_src_from(node)
-        return otk_int
-
-
 def resolve(ctx: Context, state: State, data: Any) -> Any:
     """Resolves a value of any supported type into a new value. Each type has
     its own specific handler to replace the data value."""
 
-    if isinstance(data, OtkDict):
+    if isinstance(data, dict):
         return resolve_dict(ctx, state, data)
     if isinstance(data, list):
         return resolve_list(ctx, state, data)
     if isinstance(data, str):
         return resolve_str(ctx, state, data)
-    if isinstance(data, (int, float, bool, type(None))):
+    if isinstance(data, (int, float, OtkBool, type(None))):
         return data
 
-    raise ParseTypeError(f"could not look up {type(data)} in resolvers", state)
+    raise ParseTypeError(f"could not look up {data} of type {type(data)} in resolvers", state)
 
 
 # XXX: look into this
@@ -186,16 +128,13 @@ def resolve_dict(ctx: Context, state: State, tree: dict[str, Any]) -> Any:
     return tree
 
 
-def resolve_list(ctx: Context, state: State, tree: OtkList[Any]) -> list[Any]:
+def resolve_list(ctx: Context, state: State, tree: OtkList[Any]) -> OtkList[Any]:
     """Resolving a list means applying the resolve function to each element in
     the list."""
 
     log.debug("resolving list %r", tree)
 
-    # XXX: think about finding a better way, we need to keep the OtkList
-    # here intact to be able to keep the "otk_src" information
-    tree.replace([resolve(ctx, state, val) for val in tree])
-    return tree
+    return tree.otk_dup([resolve(ctx, state, val) for val in tree])
 
 
 def resolve_str(ctx: Context, state: State, tree: str) -> Any:
@@ -306,14 +245,14 @@ def op_join(ctx: Context, state: State, tree: dict[str, Any]) -> Any:
             values[i] = substitute_vars(ctx, state, val)
 
     if all(isinstance(sl, list) for sl in values):
-        return list(itertools.chain.from_iterable(values))
+        return values.otk_dup(list(itertools.chain.from_iterable(values)))
     if all(isinstance(sl, dict) for sl in values):
         result = {}
         # XXX: this will probably need to become recursive *or* we
         # need something like a "merge_strategy" in "otk.op.join"
         for value in values:
             result.update(value)
-        return result
+        return tree.otk_dup(result)
     raise TransformDirectiveTypeError(f"cannot join {values}", state)
 
 
@@ -360,7 +299,7 @@ def substitute_vars(ctx: Context, state: State, data: str) -> Any:
                     f"expected int, float, or str but got {type(value).__name__}", state)
 
             # Replace all occurences of this name in the str
-            data = re.sub(bracket % re.escape(name), value, data)
+            data = data.otk_dup(re.sub(bracket % re.escape(name), value, data))
             log.debug("substituting %r as substring to %r", name, data)
 
     return data

test/data/error/03-target-should-not-be-list.err

@@ -1 +1 @@
-03-target-should-not-be-list.yaml: First level below a 'target' should be a dictionary (not a list)
+03-target-should-not-be-list.yaml: First level below a 'target' should be a dictionary (not a OtkList)

test/data/error/04-target-should-not-be-string.err

@@ -1 +1 @@
-First level below a 'target' should be a dictionary (not a str)
+First level below a 'target' should be a dictionary (not a OtkStr)