Adds consecutive duplicate (implicit) array support

docs/source/differences.rst

@@ -0,0 +1,79 @@
+
+.. _differences:
+
+Differences Between 010 and pfp
+===============================
+
+This section documents the known differences between pfp and 010 editor.
+
+.. toctree::
+    :maxdepth: 1
+
+Duplicate Arrays
+----------------
+
+*TLDR*: Pfp does not [yet] support non-consecutive duplicate arrays.
+Consecutive duplicate arrays are fully supported.
+
+First, some definitions and back story.
+
+Duplicate arrays are what occurs when multiple variables of the same name
+are declared in the same scope. E.g.:
+
+.. code-block:: c
+
+    int x;
+    int x;
+    if (x[0] == x[1] || x[0] == x) {
+        Printf("Same!");
+    }
+
+The 010 template script above declares ``x`` twice, creating a duplicate, or
+as pfp originally called it, an implicit array. Notice the two comparisons -
+they actually perform the same comparison:
+
+.. code-block:: c
+
+    x[0] != x[1]
+
+and
+
+.. code-block:: c
+
+    x[0] == x
+
+In 010, if the duplicate/implicit array is referenced without indexing, the
+most recently parsed field in the duplicate array is returned. I.e., it's treated
+as a normal field and not an array. However, if indexing is done on the duplicate
+array variable, the variable is treated as an array.
+
+Below is a quote on duplicate arrays from the
+`010 Editor documentation <https://www.sweetscape.com/010editor/manual/ArraysDuplicates.htm>`_:
+
+    When writing a template, regular arrays can be declaring using the same syntax
+    as scripts (see Arrays and Strings). However, 010 Editor has a syntax that
+    allows arrays to be built in a special way. When declaring template variables,
+    multiple copies of the same variable can be declared. For example:
+
+    .. code-block:: c
+
+        int x;
+        int y;
+        int x;
+
+    010 Editor allows you to treat the multiple declarations of the variable as
+    an array (this is called a Duplicate Array). In this example, x[0] could
+    be used to reference the first occurrence of x and x[1] could be used to
+    reference the second occurrence of x. Duplicate arrays can even be defined
+    with for or while loops. For example:
+
+    .. code-block:: c
+
+        local int i;
+        for( i = 0; i < 5; i++ )
+            int x;
+
+This breaks down in pfp when non-consecutive arrays are created, as is done
+in the first code sample from the 010 Editor documentation above.
+`Issue #111 <https://github.com/d0c-s4vage/pfp/issues/111>`_ tracks the effort
+to add support for non-consecutive duplicate arrays.

docs/source/index.rst

@@ -106,6 +106,12 @@ find the ``tEXt`` chunk, and change the comment: ::
             chunk.data.tEXt.comment = "NEW COMMENT"
             print("Comment after: {}".format(chunk.data.tEXt.comment))
 
+Notes
+-----
+
+A few differences do exist between 010 Editor and pfp. See the
+:ref:`differences` section for specific, documented differences.
+
 
 Contents:
 
@@ -120,6 +126,7 @@ Contents:
    interpreter
    functions
    bitstream
+   differences
 
 .. automodule:: pfp
    :members:

pfp/fields.py

@@ -662,6 +662,38 @@ class Void(Field):
     pass
 
 
+@inherit_hash
+class ImplicitArrayWrapper(Field):
+    """
+    """
+
+    last_field = None
+    implicit_array = None
+
+    def __init__(self, last_field, implicit_array):
+        """Redirect all attribute accesses to the ``last_field``, except for
+        array indexing. Array indexing is forwarded on to the
+        ``implicit_array``.
+        """
+        super(Field, self).__setattr__("last_field", last_field)
+        super(Field, self).__setattr__("implicit_array", implicit_array)
+
+    def __setattr__(self, name, value):
+        """Custom setattr that forwards all sets to the last_field
+        """
+        return setattr(self.last_field, name, value)
+
+    def __getattr__(self, name):
+        """Custom getattr that forwards all gets to last_field.
+        """
+        return getattr(self.last_field, name)
+
+    def __getitem__(self, key):
+        """Let this ImplicitArrayWrapper act like an array
+        """
+        return self.implicit_array[key]
+
+
 @inherit_hash
 class Struct(Field):
     """The struct field"""
@@ -671,6 +703,10 @@ class Struct(Field):
     _pfp__children = []
     """All children of the struct, in order added"""
 
+    _pfp__implicit_arrays = {}
+    """Mapping of all implicit arrays in this struct. All implicit arrays will
+    be resolved to a concrete array after parsing is complete"""
+
     _pfp__name_collisions = {}
     """Counters for any naming collisions"""
 
@@ -679,6 +715,8 @@ class Struct(Field):
     def __init__(self, stream=None, metadata_processor=None):
         # ordered list of children
         super(Struct, self).__setattr__("_pfp__children", [])
+        # initialize implicit arrays for this struct instance
+        super(Struct, self).__setattr__("_pfp__implicit_arrays", {})
         # for quick child access
         super(Struct, self).__setattr__("_pfp__children_map", {})
 
@@ -690,6 +728,23 @@ def __init__(self, stream=None, metadata_processor=None):
         if stream is not None:
             self._pfp__offset = stream.tell()
 
+    def _pfp__finalize(self):
+        """Finalize the results of parsing the data. Currently this involves:
+
+          * resolving implicit arrays to concrete arrays
+        """
+        to_swap = []
+        for child_name, child in six.iteritems(self._pfp__children_map):
+            if isinstance(child, Struct):
+                child._pfp__finalize()
+                continue
+            if child_name not in self._pfp__implicit_arrays:
+                continue
+            to_swap.append((child_name, child))
+
+        for child_name, child in to_swap:
+            self._pfp__children_map[child_name] = self._pfp__implicit_arrays[child_name]
+
     def _pfp__snapshot(self, recurse=True):
         """Save off the current value of the field
         """
@@ -747,7 +802,25 @@ def _pfp__add_child(self, name, child, stream=None, overwrite=False):
         ):
             return self._pfp__handle_non_consecutive_duplicate(name, child)
         elif not overwrite and name in self._pfp__children_map:
-            return self._pfp__handle_implicit_array(name, child)
+            implicit_array = self._pfp__handle_implicit_array(name, child)
+
+            # see #110 (https://github.com/d0c-s4vage/pfp/issues/110)
+            # during parsing, duplicate (implicit) arrays should always
+            # reference the last parsed variable of ``name``. However, if the
+            # variable ``name`` is indexed, then the nth duplicate array item
+            # should be returned.
+            #
+            # E.g.
+            #
+            #    int x;
+            #    int x;
+            #    int x;
+            #    Printf("%d\n", x);    // prints the latest x value
+            #    Printf("%d\n", x[0]); // prints the first x value
+            #
+            self._pfp__implicit_arrays[name] = implicit_array
+            self._pfp__children_map[name] = ImplicitArrayWrapper(child, implicit_array)
+            return child
         else:
             child._pfp__parent = self
             self._pfp__children.append(child)
@@ -812,13 +885,14 @@ def _pfp__handle_implicit_array(self, name, child):
         """Handle inserting implicit array elements
         """
         existing_child = self._pfp__children_map[name]
-        if isinstance(existing_child, Array):
+        existing_implicit_array = self._pfp__implicit_arrays.get(name, None)
+        if isinstance(existing_implicit_array, Array):
             # I don't think we should check this
             #
             # if existing_child.field_cls != child.__class__:
             #    raise errors.PfpError("implicit arrays must be sequential!")
-            existing_child.append(child)
-            return existing_child
+            existing_implicit_array.append(child)
+            return existing_implicit_array
         else:
             cls = (
                 child._pfp__class

pfp/interp.py

@@ -817,6 +817,7 @@ def parse(
         self._dlog("parsed template into ast")
 
         res = self._run(keep_successful)
+        res._pfp__finalize()
         return res
 
     def step_over(self):

tests/test_arrays.py

@@ -85,6 +85,20 @@ def test_implicit_array_basic(self):
         self.assertEqual(dom.chars[2], ord("C"))
         self.assertEqual(dom.chars[3], ord("D"))
 
+    def test_implicit_array_same_behavior_as_010(self):
+        dom = self._test_parse_build(
+            "ABCD",
+            """
+                while(!FEof()) {
+                    char x;
+                    Printf("%c", x);
+                }
+            """,
+            stdout="ABCD",
+        )
+        self.assertIsInstance(dom.x, Array)
+        self.assertEqual(dom.x, b"ABCD")
+
     def test_array_length1(self):
         dom = self._test_parse_build(
             "abcd",