From 0b4210f77a8cd8b87261829d0d6862f911e90f97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 15 Dec 2016 12:39:00 +0100
Subject: [PATCH 001/230] _parent is properly resolved in case it's used in
 value instances, because it is derived after value instances because it
 relies on type information of value instances. I try to fix this by
 collection parent-child-relation chip information during resolving user
 types, because only those can contain instances at all, and use that
 information during derivation of value instances. There's already some
 "nowClass2 in use during this process which looks a bit like a comparable
 workaround, so I'm doing the same now. Additionally, the good thing about
 this approach is that we don't need to change anything else...

I tried to find a way by resolving _parent first, but I'm not sure if this
can work at all: There's a reason value instances are derived first,
because this way they get a dataType and that dataType is needed to store
the hint to the parentClass for _parent. Without instances, where should
parent derivation store its hint? Should be pretty much only be the same
way like I do it now: Somewhere lese.

Additionally, I wonder if it's needed at all to derive _parent for
instances anymore? During resolving user types don't we already get all
possible usages of child types by parents, because simply all seq and
instances are iterated of each class and only those can refer to some
child type?
---
 .../io/kaitai/struct/ClassTypeProvider.scala  | 11 ++++++--
 .../io/kaitai/struct/TypeProcessor.scala      | 28 ++++++++++++++++---
 2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
index 861cd2df4..82c124a9d 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
@@ -5,7 +5,8 @@ import io.kaitai.struct.format._
 import io.kaitai.struct.translators.{TypeMismatchError, TypeProvider, TypeUndecidedError}
 
 class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider {
-  var nowClass = topClass
+  var nowClass           : ClassSpec         = topClass
+  var possibleParentClass: Option[ClassSpec] = None
 
   var _currentIteratorType: Option[BaseType] = None
   var _currentSwitchType: Option[BaseType] = None
@@ -21,9 +22,13 @@ class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider {
       case "_root" =>
         makeUserType(topClass)
       case "_parent" =>
-        if (inClass.parentClass == UnknownClassSpec)
+        var parent = inClass.parentClass
+        if ((parent == UnknownClassSpec) && (possibleParentClass != None))
+          parent = possibleParentClass.get
+        if (parent == UnknownClassSpec)
           throw new RuntimeException(s"Unable to derive _parent type in ${inClass.name.mkString("::")}")
-        makeUserType(inClass.parentClass)
+
+        makeUserType(parent)
       case "_io" =>
         KaitaiStreamType
       case "_" =>
diff --git a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
index 787610e98..f9626c6aa 100644
--- a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
+++ b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
@@ -5,17 +5,28 @@ import io.kaitai.struct.format._
 import io.kaitai.struct.translators.{BaseTranslator, RubyTranslator, TypeUndecidedError}
 
 import scala.collection.mutable.ListBuffer
+import scala.collection.mutable.Map
 
 object TypeProcessor {
+  // We need to record the parent types of resolved user types at their usage, which we know about
+  // on resolving user types, to be able to properly resolve "_parent" in value instances to solve
+  // some chicken&egg problem in the current implementation.
+  //
+  // The key is a string by purpose currently, because on deriving value there may be some opaque
+  // types and for some reasion their ClassSpec couldn't be queried as a key in the map without
+  // running into UnsupportedOperationException of AbstractIdentifier.toString. This doesn't happen
+  // if we provide a string using ClassSpec on our own.
+  val _parentsByChild = Map[String, ClassSpec]()
+
   def processTypes(topClass: ClassSpec): Unit = {
     // Set top class name from meta
-    topClass.name = List(topClass.meta.get.id.get)
+    topClass.name        = List(topClass.meta.get.id.get)
+    topClass.parentClass = GenericStructClassSpec
 
     markupClassNames(topClass)
     resolveUserTypes(topClass)
     deriveValueTypes(topClass)
     markupParentTypes(topClass)
-    topClass.parentClass = GenericStructClassSpec
   }
 
   // ==================================================================
@@ -52,7 +63,9 @@ object TypeProcessor {
 //    Console.println(s"deriveValueType(${curClass.name.mkString("::")})")
     var hasChanged = false
 
-    provider.nowClass = curClass
+    provider.nowClass            = curClass
+    provider.possibleParentClass = _parentsByChild.get(curClass.name.mkString("::"))
+
     curClass.instances.foreach {
       case (instName, inst) =>
         inst match {
@@ -139,8 +152,15 @@ object TypeProcessor {
       case None =>
         // Type definition not found - generate special "opaque placeholder" ClassSpec
         Some(ClassSpec.opaquePlaceholder(typeName))
-      case Some(x) =>
+      case Some(child) => {
+        // We might be called multiple times, but only have two situations: It's either always the
+        // same context or in case of different/incompatible ones there's some chance that the last
+        // call is the proper one, so we simply overwrite mappings. Depending on the target language
+        // in case of multiple incompatible usages compiler errors occur or such and things can be
+        // fixed as needed.
+        _parentsByChild(child.name.mkString("::")) = curClass
         res
+      }
     }
   }
 

From 541b0e7abc9a4aa5623532da7686b637d266c48f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 16 Dec 2016 21:21:37 +0100
Subject: [PATCH 002/230] fixed code style

---
 .../src/main/scala/io/kaitai/struct/ClassTypeProvider.scala   | 2 +-
 shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
index 82c124a9d..05184b344 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
@@ -5,7 +5,7 @@ import io.kaitai.struct.format._
 import io.kaitai.struct.translators.{TypeMismatchError, TypeProvider, TypeUndecidedError}
 
 class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider {
-  var nowClass           : ClassSpec         = topClass
+  var nowClass : ClassSpec = topClass
   var possibleParentClass: Option[ClassSpec] = None
 
   var _currentIteratorType: Option[BaseType] = None
diff --git a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
index f9626c6aa..a217d4661 100644
--- a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
+++ b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
@@ -20,7 +20,7 @@ object TypeProcessor {
 
   def processTypes(topClass: ClassSpec): Unit = {
     // Set top class name from meta
-    topClass.name        = List(topClass.meta.get.id.get)
+    topClass.name = List(topClass.meta.get.id.get)
     topClass.parentClass = GenericStructClassSpec
 
     markupClassNames(topClass)
@@ -63,7 +63,7 @@ object TypeProcessor {
 //    Console.println(s"deriveValueType(${curClass.name.mkString("::")})")
     var hasChanged = false
 
-    provider.nowClass            = curClass
+    provider.nowClass = curClass
     provider.possibleParentClass = _parentsByChild.get(curClass.name.mkString("::"))
 
     curClass.instances.foreach {

From 7f810d949c4c5397c86aaa50c53236778d86c637 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 16 Dec 2016 22:12:43 +0100
Subject: [PATCH 003/230] Changed naming scheme accorindg
 https://github.com/kaitai-io/kaitai_struct_compiler/pull/49#pullrequestreview-13411241

---
 shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
index a217d4661..e033f296f 100644
--- a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
+++ b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
@@ -16,7 +16,7 @@ object TypeProcessor {
   // types and for some reasion their ClassSpec couldn't be queried as a key in the map without
   // running into UnsupportedOperationException of AbstractIdentifier.toString. This doesn't happen
   // if we provide a string using ClassSpec on our own.
-  val _parentsByChild = Map[String, ClassSpec]()
+  val parentsByChild = Map[String, ClassSpec]()
 
   def processTypes(topClass: ClassSpec): Unit = {
     // Set top class name from meta
@@ -64,7 +64,7 @@ object TypeProcessor {
     var hasChanged = false
 
     provider.nowClass = curClass
-    provider.possibleParentClass = _parentsByChild.get(curClass.name.mkString("::"))
+    provider.possibleParentClass = parentsByChild.get(curClass.name.mkString("::"))
 
     curClass.instances.foreach {
       case (instName, inst) =>
@@ -158,7 +158,7 @@ object TypeProcessor {
         // call is the proper one, so we simply overwrite mappings. Depending on the target language
         // in case of multiple incompatible usages compiler errors occur or such and things can be
         // fixed as needed.
-        _parentsByChild(child.name.mkString("::")) = curClass
+        parentsByChild(child.name.mkString("::")) = curClass
         res
       }
     }

From 671656d343d92ce3fe31e036f9c4230481f287ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Sat, 17 Dec 2016 09:54:23 +0100
Subject: [PATCH 004/230] Code style

---
 shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
index 05184b344..8df1d15ab 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
@@ -5,7 +5,7 @@ import io.kaitai.struct.format._
 import io.kaitai.struct.translators.{TypeMismatchError, TypeProvider, TypeUndecidedError}
 
 class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider {
-  var nowClass : ClassSpec = topClass
+  var nowClass: ClassSpec = topClass
   var possibleParentClass: Option[ClassSpec] = None
 
   var _currentIteratorType: Option[BaseType] = None

From 648f49a6af20367ed0199b316d8798436fa2fa76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 19 Dec 2016 11:20:12 +0100
Subject: [PATCH 005/230] In case of multiple boolean operations as condition
 of a switch statement, parenthesis weren't properly generated. This at least
 in Java created code which didn't compile.

---
 .../io/kaitai/struct/translators/BaseTranslator.scala     | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 8f34c75a0..0a860aca8 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -172,8 +172,12 @@ abstract class BaseTranslator(val provider: TypeProvider) {
   }
 
   def doBooleanOp(op: Ast.boolop, values: Seq[Ast.expr]): String = {
-    val opStr = s" ${booleanOp(op)} "
-    values.map(translate).mkString(opStr)
+    val opStr = s"${booleanOp(op)}"
+    val dividerStr = s" ) ${opStr} ( "
+    val valuesStr = values.map(translate).mkString("( ", dividerStr, " )")
+
+    // Improve compatibility for statements like: ( ... && ... || ... ) ? ... : ...
+    s" ( ${valuesStr} ) "
   }
 
   def booleanOp(op: Ast.boolop) = op match {

From 458f3430da945e2d608665a28cf4ae09446ed3e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 19 Dec 2016 11:37:10 +0100
Subject: [PATCH 006/230] Reduce the amount of spaces for paranthesis, looks
 better to read now.

---
 .../scala/io/kaitai/struct/translators/BaseTranslator.scala | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 0a860aca8..4e106d385 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -173,11 +173,11 @@ abstract class BaseTranslator(val provider: TypeProvider) {
 
   def doBooleanOp(op: Ast.boolop, values: Seq[Ast.expr]): String = {
     val opStr = s"${booleanOp(op)}"
-    val dividerStr = s" ) ${opStr} ( "
-    val valuesStr = values.map(translate).mkString("( ", dividerStr, " )")
+    val dividerStr = s") ${opStr} ("
+    val valuesStr = values.map(translate).mkString("(", dividerStr, ")")
 
     // Improve compatibility for statements like: ( ... && ... || ... ) ? ... : ...
-    s" ( ${valuesStr} ) "
+    s" (${valuesStr}) "
   }
 
   def booleanOp(op: Ast.boolop) = op match {

From 219e152936b71f43397d9bad7321897eadaf0ead Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 20 Dec 2016 13:06:10 +0100
Subject: [PATCH 007/230] Value instances using enums are always mapped to
 EnumById, even if the expression of the value instance resolves to
 EnumByLabel. Resolving enums by ID is implemented in all languages
 individually and doesn't work e.g. in case of Java for enums itself, really
 only for numerical values.

At least some statements of value don't need any special handling at all, because they already resolve to an enum itself. But using those with EnumById would simply break things otherwise working. So I added an exception for my case using a ternary operator resolving to EnumByLabel in both cases.
---
 .../scala/io/kaitai/struct/format/InstanceSpec.scala | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
index d4d4cad29..5ed88ffb2 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
@@ -20,12 +20,20 @@ object InstanceSpec {
         // value instance
         // TODO: check conflicts with all other keys
 
-        // Wrap everything in EnumById if "enum" is used
+        // Wrap everything in EnumById if "enum" is used and "value" contains an expression using
+        // integers, but take care of exceptions like "EnumByLabel" in some situations as well.
         val value2 = ParseUtils.getOptValueStr(srcMap, "enum", path) match {
           case None =>
             value
           case Some(enumName) =>
-            Ast.expr.EnumById(Ast.identifier(enumName), value)
+            value match {
+              case Ast.expr.IfExp(_, ifTrue: Ast.expr, _) =>
+                ifTrue match {
+                  case Ast.expr.EnumByLabel(_,_) => value
+                  case _ => Ast.expr.EnumById(Ast.identifier(enumName), value)
+                }
+              case _ => Ast.expr.EnumById(Ast.identifier(enumName), value)
+            }
         }
 
         val ifExpr = ParseUtils.getOptValueStr(srcMap, "if", path).map(Expressions.parse)

From 53d668f521209c40958cbdda2efef31dc0572939 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 20 Dec 2016 15:55:43 +0100
Subject: [PATCH 008/230] I thought that mapping a value instance to an enum
 doesn't work properly, because I got invalid Java code which always used
 Enum.byId, even if I was already providing two enum instances in the value
 instance. So I've added some code in InstanceSpec to recognize this case and
 dont' use Enum.byId.

That was stupid, because the enum keyword is used to map numerical values to enums. If I know I'm already using enum instances, I simply need to not use that keyword and the type detection for the value instance already recognizes the correct enum types even in my "... ? ... : ...".

So I'm reverting the former changes to get inline with the upstream codebase again.
---
 .../scala/io/kaitai/struct/format/InstanceSpec.scala | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
index 5ed88ffb2..d4d4cad29 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
@@ -20,20 +20,12 @@ object InstanceSpec {
         // value instance
         // TODO: check conflicts with all other keys
 
-        // Wrap everything in EnumById if "enum" is used and "value" contains an expression using
-        // integers, but take care of exceptions like "EnumByLabel" in some situations as well.
+        // Wrap everything in EnumById if "enum" is used
         val value2 = ParseUtils.getOptValueStr(srcMap, "enum", path) match {
           case None =>
             value
           case Some(enumName) =>
-            value match {
-              case Ast.expr.IfExp(_, ifTrue: Ast.expr, _) =>
-                ifTrue match {
-                  case Ast.expr.EnumByLabel(_,_) => value
-                  case _ => Ast.expr.EnumById(Ast.identifier(enumName), value)
-                }
-              case _ => Ast.expr.EnumById(Ast.identifier(enumName), value)
-            }
+            Ast.expr.EnumById(Ast.identifier(enumName), value)
         }
 
         val ifExpr = ParseUtils.getOptValueStr(srcMap, "if", path).map(Expressions.parse)

From e868a53937200a12e4bd8303174d9d106a0ec5c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Wed, 4 Jan 2017 20:39:00 +0100
Subject: [PATCH 009/230] Reduce IOException in instances by checking if an
 instance is a parse instance and use try/catch to rethrow the exception as a
 runtime one. This way the always present "throws IOException" can be removed
 without the need to check all expressions of all instances on their own if
 they throw or not. Instances are often used as simple getters without any
 parsing at all and in such cases "throws IOException" has a big influence on
 the calling code, which needs to handle this exception or rethrow itself to
 maybe multiple layers of an app.

This currently doesn't handle accesses to KaitaiStream.eof and such in value instances, which throw IOException again, properly!
---
 .../io/kaitai/struct/ClassCompiler.scala      |  2 +-
 .../struct/languages/CSharpCompiler.scala     |  2 +-
 .../kaitai/struct/languages/CppCompiler.scala |  2 +-
 .../struct/languages/JavaCompiler.scala       | 23 +++++++++++++++++--
 .../struct/languages/JavaScriptCompiler.scala |  2 +-
 .../kaitai/struct/languages/PHPCompiler.scala |  2 +-
 .../struct/languages/PerlCompiler.scala       |  2 +-
 .../struct/languages/PythonCompiler.scala     |  2 +-
 .../struct/languages/RubyCompiler.scala       |  2 +-
 .../components/LanguageCompiler.scala         |  2 +-
 10 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index 0fbf65d1f..ec664ca76 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -106,7 +106,7 @@ class ClassCompiler(val topClass: ClassSpec, val lang: LanguageCompiler) extends
 
     instSpec.doc.foreach((doc) => lang.attributeDoc(instName, doc))
     lang.instanceHeader(className, instName, dataType)
-    lang.instanceCheckCacheAndReturn(instName)
+    lang.instanceCheckCacheAndReturn(instName, instSpec)
 
     instSpec match {
       case vi: ValueInstanceSpec =>
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
index 919a8f526..67607473a 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
@@ -283,7 +283,7 @@ class CSharpCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.puts("}")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
     out.puts(s"if (${flagForInstName(instName)})")
     out.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 7a5d9367f..f852ed06e 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -493,7 +493,7 @@ class CppCompiler(config: RuntimeConfig, outSrc: LanguageOutputWriter, outHdr: L
     outSrc.puts("}")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
     outSrc.puts(s"if (${calculatedFlagForName(instName)})")
     outSrc.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index f06881991..efda82f9c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -19,6 +19,8 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     with NoNeedForFullClassPath {
   import JavaCompiler._
 
+  var tryCatchForParseInstance = false
+
   override def getStatic = JavaCompiler
 
   override def universalFooter: Unit = {
@@ -352,18 +354,35 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
   }
 
   override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: BaseType): Unit = {
-    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() throws IOException {")
+    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() {")
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
     out.puts(s"if (${privateMemberName(instName)} != null)")
     out.inc
     instanceReturn(instName)
     out.dec
+
+    instSpec match {
+      case pi: ParseInstanceSpec => {
+        out.puts
+        out.puts(s"try {")
+        out.inc
+        tryCatchForParseInstance = true
+      }
+      case _ =>
+    }
   }
 
   override def instanceReturn(instName: InstanceIdentifier): Unit = {
+    if (tryCatchForParseInstance) {
+      out.puts(s"} catch (IOException ex) { throw new RuntimeException(ex); }")
+      out.puts
+      out.dec
+      tryCatchForParseInstance = false
+    }
+
     out.puts(s"return ${privateMemberName(instName)};")
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
index 2f4c86b91..9797f962a 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
@@ -335,7 +335,7 @@ class JavaScriptCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.puts("});")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
     out.puts(s"if (${privateMemberName(instName)} !== undefined)")
     out.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
index 0477098e6..c3e02213b 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
@@ -266,7 +266,7 @@ class PHPCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
     out.puts(s"if (${privateMemberName(instName)} !== null)")
     out.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
index b76e49fbe..a7636760e 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
@@ -270,7 +270,7 @@ class PerlCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.puts("my ($self) = @_;")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
     out.puts(s"return ${privateMemberName(instName)} if (${privateMemberName(instName)});")
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index 2688d8bec..7d2953188 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -218,7 +218,7 @@ class PythonCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
     out.puts(s"if hasattr(self, '${idToStr(instName)}'):")
     out.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index 5c27842c9..b57ad9486 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -278,7 +278,7 @@ class RubyCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
     out.puts(s"return ${privateMemberName(instName)} unless ${privateMemberName(instName)}.nil?")
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index e71605e8d..df140f2c3 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -100,7 +100,7 @@ abstract class LanguageCompiler(config: RuntimeConfig, out: LanguageOutputWriter
   def instanceDeclaration(attrName: InstanceIdentifier, attrType: BaseType, condSpec: ConditionalSpec) = attributeDeclaration(attrName, attrType, condSpec)
   def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: BaseType): Unit
   def instanceFooter: Unit
-  def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit
+  def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit
   def instanceReturn(instName: InstanceIdentifier): Unit
   def instanceCalculate(instName: InstanceIdentifier, dataType: BaseType, value: Ast.expr)
 

From 7d9edf591d29423d5ae513653a00153603dcbfbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Wed, 4 Jan 2017 21:15:05 +0100
Subject: [PATCH 010/230] Added recognition of KaitaiStream, because the only
 allowed metzhods on that, eof, size, pos, throw IOException as well and
 therefore can be handled like ParseInstances.

---
 .../struct/languages/JavaCompiler.scala       | 20 +++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index efda82f9c..938c61045 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -371,15 +371,31 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
         out.inc
         tryCatchForParseInstance = true
       }
-      case _ =>
+      case vi: ValueInstanceSpec => {
+        vi.value match {
+          case Ast.expr.Attribute(value: Ast.expr, attr: Ast.identifier) => {
+            val valType = translator.detectType(value)
+            valType match {
+              case KaitaiStreamType => {
+                out.puts
+                out.puts(s"try {")
+                out.inc
+                tryCatchForParseInstance = true
+              }
+              case _ =>
+            }
+          }
+          case _ =>
+        }
+      }
     }
   }
 
   override def instanceReturn(instName: InstanceIdentifier): Unit = {
     if (tryCatchForParseInstance) {
+      out.dec
       out.puts(s"} catch (IOException ex) { throw new RuntimeException(ex); }")
       out.puts
-      out.dec
       tryCatchForParseInstance = false
     }
 

From 86ba0ed6db248e79ca20bb099de65a4894d78448 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 5 Jan 2017 10:19:28 +0100
Subject: [PATCH 011/230] New implementation introducing two new callbacks, so
 that instanceCheckCacheAndReturn doesn't need to be changed for all
 languages. The current implementation already uses optional callbacks as
 well, e.g. instanceSetCalculated doesn't seem to be used in Java.

---
 .../io/kaitai/struct/ClassCompiler.scala      |  4 +-
 .../struct/languages/CSharpCompiler.scala     |  2 +-
 .../kaitai/struct/languages/CppCompiler.scala |  2 +-
 .../struct/languages/JavaCompiler.scala       | 54 +++++++++----------
 .../struct/languages/JavaScriptCompiler.scala |  2 +-
 .../kaitai/struct/languages/PHPCompiler.scala |  2 +-
 .../struct/languages/PerlCompiler.scala       |  2 +-
 .../struct/languages/PythonCompiler.scala     |  2 +-
 .../struct/languages/RubyCompiler.scala       |  2 +-
 .../components/LanguageCompiler.scala         |  4 +-
 10 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index ec664ca76..5e1f8ea98 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -106,7 +106,8 @@ class ClassCompiler(val topClass: ClassSpec, val lang: LanguageCompiler) extends
 
     instSpec.doc.foreach((doc) => lang.attributeDoc(instName, doc))
     lang.instanceHeader(className, instName, dataType)
-    lang.instanceCheckCacheAndReturn(instName, instSpec)
+    lang.instanceCheckCacheAndReturn(instName)
+    lang.instanceCalculateBegin(instSpec)
 
     instSpec match {
       case vi: ValueInstanceSpec =>
@@ -117,6 +118,7 @@ class ClassCompiler(val topClass: ClassSpec, val lang: LanguageCompiler) extends
         lang.attrParse(i, instName, extraAttrs)
     }
 
+    lang.instanceCalculateEnd(instSpec)
     lang.instanceSetCalculated(instName)
     lang.instanceReturn(instName)
     lang.instanceFooter
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
index 67607473a..919a8f526 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
@@ -283,7 +283,7 @@ class CSharpCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.puts("}")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
     out.puts(s"if (${flagForInstName(instName)})")
     out.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index f852ed06e..7a5d9367f 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -493,7 +493,7 @@ class CppCompiler(config: RuntimeConfig, outSrc: LanguageOutputWriter, outHdr: L
     outSrc.puts("}")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
     outSrc.puts(s"if (${calculatedFlagForName(instName)})")
     outSrc.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 938c61045..f15525d90 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -19,8 +19,6 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     with NoNeedForFullClassPath {
   import JavaCompiler._
 
-  var tryCatchForParseInstance = false
-
   override def getStatic = JavaCompiler
 
   override def universalFooter: Unit = {
@@ -358,47 +356,49 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
-    out.puts(s"if (${privateMemberName(instName)} != null)")
-    out.inc
-    instanceReturn(instName)
-    out.dec
-
+  def instanceCalcBodyThrows(instSpec: InstanceSpec): Boolean = {
     instSpec match {
-      case pi: ParseInstanceSpec => {
-        out.puts
-        out.puts(s"try {")
-        out.inc
-        tryCatchForParseInstance = true
-      }
+      case pi: ParseInstanceSpec => true
       case vi: ValueInstanceSpec => {
         vi.value match {
           case Ast.expr.Attribute(value: Ast.expr, attr: Ast.identifier) => {
-            val valType = translator.detectType(value)
-            valType match {
-              case KaitaiStreamType => {
-                out.puts
-                out.puts(s"try {")
-                out.inc
-                tryCatchForParseInstance = true
-              }
-              case _ =>
+            translator.detectType(value) match {
+              case KaitaiStreamType => true
+              case _ => false
             }
           }
-          case _ =>
+          case _ => false
         }
       }
+      case _ => false
     }
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
-    if (tryCatchForParseInstance) {
+  override def instanceCalculateBegin(instSpec: InstanceSpec): Unit = {
+    if (instanceCalcBodyThrows(instSpec))
+    {
+      out.puts
+      out.puts(s"try {")
+      out.inc
+    }
+  }
+
+  override def instanceCalculateEnd(instSpec: InstanceSpec): Unit = {
+    if (instanceCalcBodyThrows(instSpec)) {
       out.dec
       out.puts(s"} catch (IOException ex) { throw new RuntimeException(ex); }")
       out.puts
-      tryCatchForParseInstance = false
     }
+  }
+
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+    out.puts(s"if (${privateMemberName(instName)} != null)")
+    out.inc
+    instanceReturn(instName)
+    out.dec
+  }
 
+  override def instanceReturn(instName: InstanceIdentifier): Unit = {
     out.puts(s"return ${privateMemberName(instName)};")
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
index 9797f962a..2f4c86b91 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
@@ -335,7 +335,7 @@ class JavaScriptCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.puts("});")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
     out.puts(s"if (${privateMemberName(instName)} !== undefined)")
     out.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
index c3e02213b..0477098e6 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
@@ -266,7 +266,7 @@ class PHPCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
     out.puts(s"if (${privateMemberName(instName)} !== null)")
     out.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
index a7636760e..b76e49fbe 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
@@ -270,7 +270,7 @@ class PerlCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.puts("my ($self) = @_;")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
     out.puts(s"return ${privateMemberName(instName)} if (${privateMemberName(instName)});")
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index 7d2953188..2688d8bec 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -218,7 +218,7 @@ class PythonCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
     out.puts(s"if hasattr(self, '${idToStr(instName)}'):")
     out.inc
     instanceReturn(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index b57ad9486..5c27842c9 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -278,7 +278,7 @@ class RubyCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
     out.puts(s"return ${privateMemberName(instName)} unless ${privateMemberName(instName)}.nil?")
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index df140f2c3..715f52d0c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -96,11 +96,13 @@ abstract class LanguageCompiler(config: RuntimeConfig, out: LanguageOutputWriter
   def popPos(io: String): Unit
 
   def instanceClear(instName: InstanceIdentifier): Unit = {}
+  def instanceCalculateBegin(instSpec: InstanceSpec): Unit = {}
+  def instanceCalculateEnd(instSpec: InstanceSpec): Unit = {}
   def instanceSetCalculated(instName: InstanceIdentifier): Unit = {}
   def instanceDeclaration(attrName: InstanceIdentifier, attrType: BaseType, condSpec: ConditionalSpec) = attributeDeclaration(attrName, attrType, condSpec)
   def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: BaseType): Unit
   def instanceFooter: Unit
-  def instanceCheckCacheAndReturn(instName: InstanceIdentifier, instSpec: InstanceSpec): Unit
+  def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit
   def instanceReturn(instName: InstanceIdentifier): Unit
   def instanceCalculate(instName: InstanceIdentifier, dataType: BaseType, value: Ast.expr)
 

From d62a4396e1f7172ad4209c24f158dc2e52b95bed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 5 Jan 2017 12:56:37 +0100
Subject: [PATCH 012/230] Improved detection of throwing expressions in case of
 combined expressions, like "_io.eof() || ...", which were not recognized
 before.

---
 .../struct/languages/JavaCompiler.scala       | 45 ++++++++++++++-----
 1 file changed, 33 insertions(+), 12 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index f15525d90..b17e2744e 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -356,20 +356,39 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.inc
   }
 
-  def instanceCalcBodyThrows(instSpec: InstanceSpec): Boolean = {
-    instSpec match {
-      case pi: ParseInstanceSpec => true
-      case vi: ValueInstanceSpec => {
-        vi.value match {
-          case Ast.expr.Attribute(value: Ast.expr, attr: Ast.identifier) => {
-            translator.detectType(value) match {
-              case KaitaiStreamType => true
-              case _ => false
+  def detectExprThrows(value: Ast.expr): Boolean = {
+    value match {
+      case Ast.expr.UnaryOp(_, v: Ast.expr)                     => detectExprThrows(v)
+      case Ast.expr.Compare(left: Ast.expr, _, right: Ast.expr) => detectExprThrows(left) || detectExprThrows(right)
+      case Ast.expr.BinOp(  left: Ast.expr, _, right: Ast.expr) => detectExprThrows(left) || detectExprThrows(right)
+      case Ast.expr.BoolOp(_, values: Seq[Ast.expr])            => detectExprThrows(values)
+      case Ast.expr.Call(func: Ast.expr, args: Seq[Ast.expr])   => detectExprThrows(args) || detectExprThrows(func)
+      case Ast.expr.Attribute(value: Ast.expr, attr: Ast.identifier) =>
+        val valType = translator.detectType(value)
+        valType match {
+          case KaitaiStreamType =>
+            attr.name match {
+              case "size" => true
+              case "eof"  => true
+              case "pos"  => true
+              case _      => false
             }
-          }
           case _ => false
         }
-      }
+      case _ => false
+    }
+  }
+
+  def detectExprThrows(values: Seq[Ast.expr]): Boolean = {
+    values.map(   value => detectExprThrows(value))
+          .filter(value => value == true)
+          .size > 0
+  }
+
+  def instanceCalcBodyThrows(instSpec: InstanceSpec): Boolean = {
+    instSpec match {
+      case pi: ParseInstanceSpec => true
+      case vi: ValueInstanceSpec => detectExprThrows(vi.value)
       case _ => false
     }
   }
@@ -386,7 +405,9 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
   override def instanceCalculateEnd(instSpec: InstanceSpec): Unit = {
     if (instanceCalcBodyThrows(instSpec)) {
       out.dec
-      out.puts(s"} catch (IOException ex) { throw new RuntimeException(ex); }")
+      out.puts(s"} catch (IOException ex) {")
+      out.inc
+      out.puts(s"throw new RuntimeException(ex); }")
       out.puts
     }
   }

From 599cfbc3922a88dd2238c3dde112223ddd790630 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 5 Jan 2017 15:11:46 +0100
Subject: [PATCH 013/230] Reverted my changes after creating a branch to try an
 alternative implementation.

---
 .../io/kaitai/struct/ClassCompiler.scala      |  2 -
 .../struct/languages/JavaCompiler.scala       | 58 +------------------
 .../components/LanguageCompiler.scala         |  2 -
 3 files changed, 1 insertion(+), 61 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index 5e1f8ea98..0fbf65d1f 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -107,7 +107,6 @@ class ClassCompiler(val topClass: ClassSpec, val lang: LanguageCompiler) extends
     instSpec.doc.foreach((doc) => lang.attributeDoc(instName, doc))
     lang.instanceHeader(className, instName, dataType)
     lang.instanceCheckCacheAndReturn(instName)
-    lang.instanceCalculateBegin(instSpec)
 
     instSpec match {
       case vi: ValueInstanceSpec =>
@@ -118,7 +117,6 @@ class ClassCompiler(val topClass: ClassSpec, val lang: LanguageCompiler) extends
         lang.attrParse(i, instName, extraAttrs)
     }
 
-    lang.instanceCalculateEnd(instSpec)
     lang.instanceSetCalculated(instName)
     lang.instanceReturn(instName)
     lang.instanceFooter
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index b17e2744e..f06881991 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -352,66 +352,10 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
   }
 
   override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: BaseType): Unit = {
-    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() {")
+    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() throws IOException {")
     out.inc
   }
 
-  def detectExprThrows(value: Ast.expr): Boolean = {
-    value match {
-      case Ast.expr.UnaryOp(_, v: Ast.expr)                     => detectExprThrows(v)
-      case Ast.expr.Compare(left: Ast.expr, _, right: Ast.expr) => detectExprThrows(left) || detectExprThrows(right)
-      case Ast.expr.BinOp(  left: Ast.expr, _, right: Ast.expr) => detectExprThrows(left) || detectExprThrows(right)
-      case Ast.expr.BoolOp(_, values: Seq[Ast.expr])            => detectExprThrows(values)
-      case Ast.expr.Call(func: Ast.expr, args: Seq[Ast.expr])   => detectExprThrows(args) || detectExprThrows(func)
-      case Ast.expr.Attribute(value: Ast.expr, attr: Ast.identifier) =>
-        val valType = translator.detectType(value)
-        valType match {
-          case KaitaiStreamType =>
-            attr.name match {
-              case "size" => true
-              case "eof"  => true
-              case "pos"  => true
-              case _      => false
-            }
-          case _ => false
-        }
-      case _ => false
-    }
-  }
-
-  def detectExprThrows(values: Seq[Ast.expr]): Boolean = {
-    values.map(   value => detectExprThrows(value))
-          .filter(value => value == true)
-          .size > 0
-  }
-
-  def instanceCalcBodyThrows(instSpec: InstanceSpec): Boolean = {
-    instSpec match {
-      case pi: ParseInstanceSpec => true
-      case vi: ValueInstanceSpec => detectExprThrows(vi.value)
-      case _ => false
-    }
-  }
-
-  override def instanceCalculateBegin(instSpec: InstanceSpec): Unit = {
-    if (instanceCalcBodyThrows(instSpec))
-    {
-      out.puts
-      out.puts(s"try {")
-      out.inc
-    }
-  }
-
-  override def instanceCalculateEnd(instSpec: InstanceSpec): Unit = {
-    if (instanceCalcBodyThrows(instSpec)) {
-      out.dec
-      out.puts(s"} catch (IOException ex) {")
-      out.inc
-      out.puts(s"throw new RuntimeException(ex); }")
-      out.puts
-    }
-  }
-
   override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
     out.puts(s"if (${privateMemberName(instName)} != null)")
     out.inc
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index 715f52d0c..e71605e8d 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -96,8 +96,6 @@ abstract class LanguageCompiler(config: RuntimeConfig, out: LanguageOutputWriter
   def popPos(io: String): Unit
 
   def instanceClear(instName: InstanceIdentifier): Unit = {}
-  def instanceCalculateBegin(instSpec: InstanceSpec): Unit = {}
-  def instanceCalculateEnd(instSpec: InstanceSpec): Unit = {}
   def instanceSetCalculated(instName: InstanceIdentifier): Unit = {}
   def instanceDeclaration(attrName: InstanceIdentifier, attrType: BaseType, condSpec: ConditionalSpec) = attributeDeclaration(attrName, attrType, condSpec)
   def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: BaseType): Unit

From f1e6221d750e9b3310afeb033d425e636b6d0583 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 5 Jan 2017 15:38:34 +0100
Subject: [PATCH 014/230] Check for needed IOExceptions under the premise that
 Runtime doesn't throw unnecessary IOExceptions anymore, in which case such an
 Exception should only be thrown in case a user type is parsed.

---
 .../scala/io/kaitai/struct/languages/JavaCompiler.scala    | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index f06881991..1f538248d 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -352,7 +352,12 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
   }
 
   override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: BaseType): Unit = {
-    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() throws IOException {")
+    val throwsClause = dataType match {
+      case t: UserType => s" throws IOException"
+      case _ => s""
+    }
+
+    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}()${throwsClause} {")
     out.inc
   }
 

From a2938fef80e9f9c01d65301566a69c86f5e83f2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 5 Jan 2017 15:39:39 +0100
Subject: [PATCH 015/230] Reverting to try another implementation where all
 cases of IOExceptions are removed which are not necessary and only were her
 ebecause runtime throwd those unnecessary ones.

---
 .../scala/io/kaitai/struct/languages/JavaCompiler.scala    | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 1f538248d..f06881991 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -352,12 +352,7 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
   }
 
   override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: BaseType): Unit = {
-    val throwsClause = dataType match {
-      case t: UserType => s" throws IOException"
-      case _ => s""
-    }
-
-    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}()${throwsClause} {")
+    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() throws IOException {")
     out.inc
   }
 

From 9104c7d5265a14c6fd701a55efb710eaa8ad72c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 5 Jan 2017 16:24:02 +0100
Subject: [PATCH 016/230] With removed unnecessary IOExceptions in the runtime,
 a lot of the IOExceptions here can be removed as well and make user's life a
 lot easier, because one doesn't have to care about catching or throwing all
 those exceptions introduced using parse instances, some value instances etc.
 at all anymore. There are only very few places now where IOException is
 needed and useful.

---
 .../io/kaitai/struct/languages/JavaCompiler.scala      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index f06881991..534fc597c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -72,7 +72,7 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
 
   override def classConstructorHeader(name: String, parentClassName: String, rootClassName: String): Unit = {
     out.puts
-    out.puts(s"public ${type2class(name)}($kstreamName _io) throws IOException {")
+    out.puts(s"public ${type2class(name)}($kstreamName _io) {")
     out.inc
     out.puts("super(_io);")
     if (name == rootClassName)
@@ -83,7 +83,7 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.puts("}")
 
     out.puts
-    out.puts(s"public ${type2class(name)}($kstreamName _io, ${type2class(parentClassName)} _parent) throws IOException {")
+    out.puts(s"public ${type2class(name)}($kstreamName _io, ${type2class(parentClassName)} _parent) {")
     out.inc
     out.puts("super(_io);")
     out.puts("this._parent = _parent;")
@@ -95,7 +95,7 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     out.puts("}")
 
     out.puts
-    out.puts(s"public ${type2class(name)}($kstreamName _io, ${type2class(parentClassName)} _parent, ${type2class(rootClassName)} _root) throws IOException {")
+    out.puts(s"public ${type2class(name)}($kstreamName _io, ${type2class(parentClassName)} _parent, ${type2class(rootClassName)} _root) {")
     out.inc
     out.puts("super(_io);")
     out.puts("this._parent = _parent;")
@@ -110,7 +110,7 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
     } else {
       "private"
     }
-    out.puts(s"$readAccessAndType void _read() throws IOException {")
+    out.puts(s"$readAccessAndType void _read() {")
     out.inc
   }
 
@@ -352,7 +352,7 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
   }
 
   override def instanceHeader(className: String, instName: InstanceIdentifier, dataType: BaseType): Unit = {
-    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() throws IOException {")
+    out.puts(s"public ${kaitaiType2JavaTypeBoxed(dataType)} ${idToStr(instName)}() {")
     out.inc
   }
 

From 0b4a1f75fbf7052b8b5302103d25a9ffb105a48c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 9 Jan 2017 10:35:33 +0100
Subject: [PATCH 017/230] Implemented support to process packed BCD to a string
 or decimal representation.

---
 .../scala/io/kaitai/struct/format/AttrSpec.scala   |  2 +-
 .../io/kaitai/struct/format/ProcessExpr.scala      | 14 +++++++++++++-
 .../io/kaitai/struct/languages/JavaCompiler.scala  |  8 ++++++++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
index 351331285..5b59a9612 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
@@ -90,7 +90,7 @@ object AttrSpec {
 
   def fromYaml(srcMap: Map[String, Any], path: List[String], metaDef: MetaDefaults, id: Identifier): AttrSpec = {
     val doc = ParseUtils.getOptValueStr(srcMap, "doc", path)
-    val process = ProcessExpr.fromStr(ParseUtils.getOptValueStr(srcMap, "process", path)) // TODO: add proper path propagation
+    val process = ProcessExpr.fromStr(ParseUtils.getOptValueStr(srcMap, "process", path), metaDef) // TODO: add proper path propagation
     val contents = srcMap.get("contents").map(parseContentSpec(_, path ++ List("contents")))
     val size = ParseUtils.getOptValueStr(srcMap, "size", path).map(Expressions.parse)
     val sizeEos = ParseUtils.getOptValueBool(srcMap, "size-eos", path).getOrElse(false)
diff --git a/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala b/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
index 3f56f2b50..09d822c07 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
@@ -1,6 +1,7 @@
 package io.kaitai.struct.format
 
 import io.kaitai.struct.exprlang.{Expressions, Ast}
+import io.kaitai.struct.exprlang.DataType.Endianness
 
 trait ProcessExpr {
   def outputType: String
@@ -18,12 +19,19 @@ case class ProcessXor(key: Ast.expr) extends ProcessExpr {
 case class ProcessRotate(left: Boolean, key: Ast.expr) extends ProcessExpr {
   override def outputType: String = null
 }
+case class ProcessBcdToStr(endian: Option[Endianness], ltr: Boolean) extends ProcessExpr {
+  override def outputType: String = null
+}
+case class ProcessBcdToDecimal(endian: Option[Endianness]) extends ProcessExpr {
+  override def outputType: String = null
+}
 
 object ProcessExpr {
   private val ReXor = "^xor\\(\\s*(.*?)\\s*\\)$".r
   private val ReRotate = "^ro(l|r)\\(\\s*(.*?)\\s*\\)$".r
+  private val ReBcdToStr = "^bcd_to_str\\(\\s*((?:ltr|rtl))\\s*\\)$".r
 
-  def fromStr(s: Option[String]): Option[ProcessExpr] = {
+  def fromStr(s: Option[String], metaDef: MetaDefaults): Option[ProcessExpr] = {
     s match {
       case None =>
         None
@@ -37,6 +45,10 @@ object ProcessExpr {
             ProcessXor(Expressions.parse(arg))
           case ReRotate(dir, arg) =>
             ProcessRotate(dir == "l", Expressions.parse(arg))
+          case ReBcdToStr(ltr) =>
+            ProcessBcdToStr(metaDef.endian, ltr == "ltr")
+          case "bcd_to_decimal" =>
+            ProcessBcdToDecimal(metaDef.endian)
           case _ =>
             throw new RuntimeException(s"Invalid process: '$s'")
         })
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 534fc597c..0b81db5fd 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -153,6 +153,14 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
           s"8 - (${expression(rotValue)})"
         }
         out.puts(s"$destName = $kstreamName.processRotateLeft($srcName, $expr, 1);")
+      case ProcessBcdToStr(endian, ltr) =>
+        val le = endian match {
+          case Some(BigEndian) => false
+          case _ => true
+        }
+        out.puts(s"$destName = $kstreamName.processBcdToStr($srcName, le, ltr);")
+      case ProcessBcdToDecimal(endian) =>
+        out.puts(s"$destName = $kstreamName.processBcdToDecimal($srcName, le);")
     }
   }
 

From 3e0fd52aca129b62a7ecdd9f0d0abc3a39de95c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 9 Jan 2017 10:44:44 +0100
Subject: [PATCH 018/230] Args were not generated properly.

---
 .../kaitai/struct/languages/JavaCompiler.scala  | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 0b81db5fd..1a74588ca 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -154,13 +154,20 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
         }
         out.puts(s"$destName = $kstreamName.processRotateLeft($srcName, $expr, 1);")
       case ProcessBcdToStr(endian, ltr) =>
-        val le = endian match {
-          case Some(BigEndian) => false
-          case _ => true
+        val isLeStr = endian match {
+          case Some(BigEndian) => translator.doBoolLiteral(false)
+          case _ => translator.doBoolLiteral(true)
         }
-        out.puts(s"$destName = $kstreamName.processBcdToStr($srcName, le, ltr);")
+        val needsLtrStr = translator.doBoolLiteral(ltr)
+
+        out.puts(s"$destName = $kstreamName.processBcdToStr($srcName, ${isLeStr}, ${needsLtrStr});")
       case ProcessBcdToDecimal(endian) =>
-        out.puts(s"$destName = $kstreamName.processBcdToDecimal($srcName, le);")
+        val isLeStr = endian match {
+            case Some(BigEndian) => translator.doBoolLiteral(false)
+            case _ => translator.doBoolLiteral(true)
+          }
+
+        out.puts(s"$destName = $kstreamName.processBcdToDecimal($srcName, ${isLeStr});")
     }
   }
 

From 479e8f69bc55c28aec70afef483dc8e615c4ac87 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 9 Jan 2017 14:41:31 +0100
Subject: [PATCH 019/230] Added support for bcd_to_str and _decimal in Ruby.

---
 .../io/kaitai/struct/languages/RubyCompiler.scala | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index 5c27842c9..6de355554 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -107,6 +107,21 @@ class RubyCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
           s"8 - (${expression(rotValue)})"
         }
         s"$destName = $kstreamName::process_rotate_left($srcName, $expr, 1)"
+      case ProcessBcdToStr(endian, ltr) =>
+        val isLeStr = endian match {
+          case Some(BigEndian) => translator.doBoolLiteral(false)
+          case _ => translator.doBoolLiteral(true)
+        }
+        val needsLtrStr = translator.doBoolLiteral(ltr)
+
+        s"$destName = $kstreamName::process_bcd_to_str($srcName, ${isLeStr}, ${needsLtrStr})"
+      case ProcessBcdToDecimal(endian) =>
+        val isLeStr = endian match {
+            case Some(BigEndian) => translator.doBoolLiteral(false)
+            case _ => translator.doBoolLiteral(true)
+          }
+
+        s"$destName = $kstreamName::process_bcd_to_decimal($srcName, ${isLeStr})"
     })
   }
 

From 69458eeb41081f538536d22862e91a47017cd05e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 10 Jan 2017 12:40:05 +0100
Subject: [PATCH 020/230] Reverted changes for supporting process statements
 with bcd-encoding, because it doesn't look like my changes going to be
 accepted soon and I don't want to use features not most likely available
 "officially" as well. @GreyCat provided an alternate solution using a user
 type which might even be better in the long term for what I need, because I
 can access individual digits and such.

---
 .../scala/io/kaitai/struct/format/AttrSpec.scala  |  2 +-
 .../io/kaitai/struct/format/ProcessExpr.scala     | 14 +-------------
 .../io/kaitai/struct/languages/JavaCompiler.scala | 15 ---------------
 .../io/kaitai/struct/languages/RubyCompiler.scala | 15 ---------------
 4 files changed, 2 insertions(+), 44 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
index 5b59a9612..351331285 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
@@ -90,7 +90,7 @@ object AttrSpec {
 
   def fromYaml(srcMap: Map[String, Any], path: List[String], metaDef: MetaDefaults, id: Identifier): AttrSpec = {
     val doc = ParseUtils.getOptValueStr(srcMap, "doc", path)
-    val process = ProcessExpr.fromStr(ParseUtils.getOptValueStr(srcMap, "process", path), metaDef) // TODO: add proper path propagation
+    val process = ProcessExpr.fromStr(ParseUtils.getOptValueStr(srcMap, "process", path)) // TODO: add proper path propagation
     val contents = srcMap.get("contents").map(parseContentSpec(_, path ++ List("contents")))
     val size = ParseUtils.getOptValueStr(srcMap, "size", path).map(Expressions.parse)
     val sizeEos = ParseUtils.getOptValueBool(srcMap, "size-eos", path).getOrElse(false)
diff --git a/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala b/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
index 09d822c07..3f56f2b50 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
@@ -1,7 +1,6 @@
 package io.kaitai.struct.format
 
 import io.kaitai.struct.exprlang.{Expressions, Ast}
-import io.kaitai.struct.exprlang.DataType.Endianness
 
 trait ProcessExpr {
   def outputType: String
@@ -19,19 +18,12 @@ case class ProcessXor(key: Ast.expr) extends ProcessExpr {
 case class ProcessRotate(left: Boolean, key: Ast.expr) extends ProcessExpr {
   override def outputType: String = null
 }
-case class ProcessBcdToStr(endian: Option[Endianness], ltr: Boolean) extends ProcessExpr {
-  override def outputType: String = null
-}
-case class ProcessBcdToDecimal(endian: Option[Endianness]) extends ProcessExpr {
-  override def outputType: String = null
-}
 
 object ProcessExpr {
   private val ReXor = "^xor\\(\\s*(.*?)\\s*\\)$".r
   private val ReRotate = "^ro(l|r)\\(\\s*(.*?)\\s*\\)$".r
-  private val ReBcdToStr = "^bcd_to_str\\(\\s*((?:ltr|rtl))\\s*\\)$".r
 
-  def fromStr(s: Option[String], metaDef: MetaDefaults): Option[ProcessExpr] = {
+  def fromStr(s: Option[String]): Option[ProcessExpr] = {
     s match {
       case None =>
         None
@@ -45,10 +37,6 @@ object ProcessExpr {
             ProcessXor(Expressions.parse(arg))
           case ReRotate(dir, arg) =>
             ProcessRotate(dir == "l", Expressions.parse(arg))
-          case ReBcdToStr(ltr) =>
-            ProcessBcdToStr(metaDef.endian, ltr == "ltr")
-          case "bcd_to_decimal" =>
-            ProcessBcdToDecimal(metaDef.endian)
           case _ =>
             throw new RuntimeException(s"Invalid process: '$s'")
         })
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 1a74588ca..534fc597c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -153,21 +153,6 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
           s"8 - (${expression(rotValue)})"
         }
         out.puts(s"$destName = $kstreamName.processRotateLeft($srcName, $expr, 1);")
-      case ProcessBcdToStr(endian, ltr) =>
-        val isLeStr = endian match {
-          case Some(BigEndian) => translator.doBoolLiteral(false)
-          case _ => translator.doBoolLiteral(true)
-        }
-        val needsLtrStr = translator.doBoolLiteral(ltr)
-
-        out.puts(s"$destName = $kstreamName.processBcdToStr($srcName, ${isLeStr}, ${needsLtrStr});")
-      case ProcessBcdToDecimal(endian) =>
-        val isLeStr = endian match {
-            case Some(BigEndian) => translator.doBoolLiteral(false)
-            case _ => translator.doBoolLiteral(true)
-          }
-
-        out.puts(s"$destName = $kstreamName.processBcdToDecimal($srcName, ${isLeStr});")
     }
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index 6de355554..5c27842c9 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -107,21 +107,6 @@ class RubyCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
           s"8 - (${expression(rotValue)})"
         }
         s"$destName = $kstreamName::process_rotate_left($srcName, $expr, 1)"
-      case ProcessBcdToStr(endian, ltr) =>
-        val isLeStr = endian match {
-          case Some(BigEndian) => translator.doBoolLiteral(false)
-          case _ => translator.doBoolLiteral(true)
-        }
-        val needsLtrStr = translator.doBoolLiteral(ltr)
-
-        s"$destName = $kstreamName::process_bcd_to_str($srcName, ${isLeStr}, ${needsLtrStr})"
-      case ProcessBcdToDecimal(endian) =>
-        val isLeStr = endian match {
-            case Some(BigEndian) => translator.doBoolLiteral(false)
-            case _ => translator.doBoolLiteral(true)
-          }
-
-        s"$destName = $kstreamName::process_bcd_to_decimal($srcName, ${isLeStr})"
     })
   }
 

From 7f1dfb77611b6f308d44eacf71cb522723aea941 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 10 Jan 2017 18:51:48 +0100
Subject: [PATCH 021/230] Implemented "to_s" for IntType to easier get a
 textual representation for BCD values, while it is of general use of course.

---
 .../struct/translators/BaseTranslator.scala   | 10 +++++-
 .../struct/translators/CSharpTranslator.scala |  2 ++
 .../struct/translators/CppTranslator.scala    |  8 +++++
 .../translators/JavaScriptTranslator.scala    |  3 ++
 .../struct/translators/JavaTranslator.scala   |  2 ++
 .../struct/translators/PHPTranslator.scala    |  5 +++
 .../struct/translators/PerlTranslator.scala   | 33 +++++++++++++------
 .../struct/translators/PythonTranslator.scala | 12 +++++++
 .../struct/translators/RubyTranslator.scala   |  3 ++
 9 files changed, 67 insertions(+), 11 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 4e106d385..acdb8abd8 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -84,7 +84,9 @@ abstract class BaseTranslator(val provider: TypeProvider) {
               case "to_i" => strToInt(value, Ast.expr.IntNum(10))
             }
           case _: IntType =>
-            throw new RuntimeException(s"don't know how to call anything on ${valType}")
+            attr.name match {
+              case "to_s" => intToStr(value, Ast.expr.IntNum(10))
+            }
           case ArrayType(inType) =>
             attr.name match {
               case "first" => arrayFirst(value)
@@ -212,6 +214,7 @@ abstract class BaseTranslator(val provider: TypeProvider) {
   // Predefined methods of various types
   def strConcat(left: Ast.expr, right: Ast.expr): String = s"${translate(left)} + ${translate(right)}"
   def strToInt(s: Ast.expr, base: Ast.expr): String
+  def intToStr(i: Ast.expr, base: Ast.expr): String
   def strLength(s: Ast.expr): String
   def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String
 
@@ -323,6 +326,11 @@ abstract class BaseTranslator(val provider: TypeProvider) {
               case "to_i" => CalcIntType
               case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
             }
+          case _: IntType =>
+            attr.name match {
+              case "to_s" => CalcStrType
+              case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+            }
           case ArrayType(inType) =>
             attr.name match {
               case "first" | "last" => inType
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
index 1cb449811..9abe0c822 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
@@ -59,6 +59,8 @@ class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider)
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String =
     s"Convert.ToInt64(${translate(s)}, ${translate(base)})"
+  override def intToStr(i: expr, base: expr): String =
+    s"Convert.ToString(${translate(i)}, ${translate(base)})"
   override def strLength(s: expr): String =
     s"${translate(s)}.Length"
   override def strSubstring(s: expr, from: expr, to: expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 294a832b2..817cdf3a7 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -61,6 +61,14 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
       case _ => s", 0, $baseStr"
     }) + ")"
   }
+  override def intToStr(i: expr, base: expr): String = {
+    val baseStr = translate(base)
+    baseStr match {
+      case "10" =>
+        s"std::to_string(${translate(i)})"
+      case _ => throw new UnsupportedOperationException(baseStr)
+    }
+  }
   override def strLength(s: expr): String =
     s"${translate(s)}.length()"
   override def strSubstring(s: expr, from: expr, to: expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
index 520064ba9..fc1573d48 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
@@ -47,6 +47,9 @@ class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provid
   override def strToInt(s: expr, base: expr): String =
     s"Number.parseInt(${translate(s)}, ${translate(base)})"
 
+  override def intToStr(i: expr, base: expr): String =
+    s"(${translate(i)}).toString(${translate(base)})"
+
   override def strLength(s: expr): String =
     s"${translate(s)}.length"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 15b615c04..f479257aa 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -62,6 +62,8 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String =
     s"Long.parseLong(${translate(s)}, ${translate(base)})"
+  override def intToStr(i: expr, base: expr): String =
+    s"Long.toString(${translate(i)}, ${translate(base)})"
   override def strLength(s: expr): String =
     s"${translate(s)}.length()"
   override def strSubstring(s: expr, from: expr, to: expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
index 4d9fd5870..27490f920 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
@@ -53,6 +53,11 @@ class PHPTranslator(provider: TypeProvider, lang: PHPCompiler) extends BaseTrans
   override def strToInt(s: expr, base: expr): String =
     s"intval(${translate(s)}, ${translate(base)})"
 
+  override def intToStr(i: expr, base: expr): String = {
+    val fromBase = Ast.expr.IntNum(10)
+    s"base_convert(strval(${translate(i)}, ${translate(fromBase)}, ${translate(base)}))"
+  }
+
   override def strLength(s: expr): String =
     s"strlen(${translate(s)})"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
index 606dffdc6..2873152b6 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
@@ -66,18 +66,31 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   override def strConcat(left: Ast.expr, right: Ast.expr): String =
     s"${translate(left)} . ${translate(right)}"
   override def strToInt(s: Ast.expr, base: Ast.expr): String = {
-    base match {
-      case Ast.expr.IntNum(baseNum) =>
-        baseNum.toInt match {
-          case 10 =>
-            translate(s)
-          case 8 =>
-            s"oct(${translate(s)})"
-          case 16 =>
-            s"hex(${translate(s)})"
-        }
+    val baseStr = translate(base)
+    baseStr match {
+      case "8" =>
+        s"oct(${translate(s)})"
+      case "10" =>
+        translate(s)
+      case "16" =>
+        s"hex(${translate(s)})"
+      case _ => throw new UnsupportedOperationException(baseStr)
     }
   }
+  override def intToStr(i: Ast.expr, base: Ast.expr): String = {
+    val baseStr = translate(base)
+    val format = baseStr match {
+      case "8" =>
+        s"%o"
+      case "10" =>
+        s"%d"
+      case "16" =>
+        s"0x%X"
+      case _ => throw new UnsupportedOperationException(baseStr)
+    }
+
+    s"sprintf('$format', ${translate(i)})"
+  }
   override def strLength(value: Ast.expr): String =
     s"length(${translate(value)})"
   override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
index 8615e3ae6..498aeec13 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
@@ -57,6 +57,18 @@ class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider)
     }
     s"int(${translate(s)}$add)"
   }
+  override def intToStr(i: Ast.expr, base: Ast.expr): String = {
+    val baseStr = translate(base)
+    val func = baseStr match {
+      case "2" => "bin"
+      case "8" => "oct"
+      case "10" => "str"
+      case "16" => "hex"
+      case _ => throw new UnsupportedOperationException(baseStr)
+    }
+
+    s"$func(${translate(i)})"
+  }
   override def strLength(value: Ast.expr): String =
     s"len(${translate(value)})"
   override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
index 6c61250d8..d63c04d0f 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -27,6 +27,9 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
       case _ => s"($baseStr)"
     })
   }
+  override def intToStr(i: Ast.expr, base: Ast.expr): String = {
+    translate(i) + s".to_s(${translate(base)})"
+  }
   override def strLength(s: Ast.expr): String =
     s"${translate(s)}.size"
   override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String =

From 6ba99f9a7dc9ddaf12babf1fbfe854490c0dd8a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 27 Jan 2017 12:57:57 +0100
Subject: [PATCH 022/230] CalcIntType -> long, because int might lead to loss
 of data in some calculations.

---
 .../scala/io/kaitai/struct/languages/JavaCompiler.scala   | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 534fc597c..63b04e1e9 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -246,8 +246,8 @@ class JavaCompiler(config: RuntimeConfig, out: LanguageOutputWriter)
 
   override def condRepeatExprHeader(id: Identifier, io: String, dataType: BaseType, needRaw: Boolean, repeatExpr: expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>((int) (${expression(repeatExpr)}));")
-    out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}((int) (${expression(repeatExpr)}));")
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>(Long.valueOf(${expression(repeatExpr)}).intValue());")
+    out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}(Long.valueOf(${expression(repeatExpr)}).intValue());")
     out.puts(s"for (int i = 0; i < ${expression(repeatExpr)}; i++) {")
     out.inc
   }
@@ -485,7 +485,7 @@ object JavaCompiler extends LanguageCompilerStatic
       case BitsType(_) => "long"
 
       case BooleanType => "boolean"
-      case CalcIntType => "int"
+      case CalcIntType => "long"
       case CalcFloatType => "double"
 
       case _: StrType => "String"
@@ -530,7 +530,7 @@ object JavaCompiler extends LanguageCompilerStatic
       case BitsType(_) => "Long"
 
       case BooleanType => "Boolean"
-      case CalcIntType => "Integer"
+      case CalcIntType => "Long"
       case CalcFloatType => "Double"
 
       case _: StrType => "String"

From c8d32974da0f7e7b04ebdc5afe190a40f59cbcfc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 27 Jan 2017 20:11:16 +0100
Subject: [PATCH 023/230] doIntLiteral couldn't handle >32 Bit signed int

I had to deal with int literals of less and more than 32 Bit signed int and in the latter case the generated code for Java didn't compile, because the compiler provided the error message that the number is too large to be an int. I needs the suffix "L" in such cases and that is now added for Java only.
---
 .../io/kaitai/struct/translators/TranslatorSpec.scala      | 5 +++++
 .../io/kaitai/struct/translators/BaseTranslator.scala      | 2 +-
 .../io/kaitai/struct/translators/JavaTranslator.scala      | 7 +++++++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 5635df332..d72a07826 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -19,6 +19,11 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks {
     everybody("1234", "1234"),
     everybody("-456", "-456"),
     everybody("0x1234", "4660"),
+    // less and more than 32 Bit signed int
+    everybody("1000000000", "1000000000"),
+    everybodyExcept("100000000000", "100000000000", Map[LanguageCompilerStatic, String](
+        JavaCompiler -> "100000000000L"
+    )),
 
     // Float literals
     everybody("1.0", "1.0", CalcFloatType),
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index acdb8abd8..9fb4ea4ff 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -197,7 +197,7 @@ abstract class BaseTranslator(val provider: TypeProvider) {
   def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String
 
   // Literals
-  def doIntLiteral(n: Any): String = n.toString
+  def doIntLiteral(n: BigInt): String = n.toString
   def doFloatLiteral(n: Any): String = n.toString
   def doStringLiteral(s: String): String = "\"" + s + "\""
   def doBoolLiteral(n: Boolean): String = n.toString
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index f479257aa..3b988f6dd 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -7,6 +7,13 @@ import io.kaitai.struct.exprlang.DataType.{BaseType, IntType}
 import io.kaitai.struct.languages.JavaCompiler
 
 class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
+  override def doIntLiteral(n: BigInt): String = {
+    val literal = n.toString
+    val suffix = if (n > Int.MaxValue) "L" else ""
+
+    s"${literal}${suffix}"
+  }
+
   override def doArrayLiteral(t: BaseType, value: Seq[expr]): String = {
     val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t)
     val commaStr = value.map((v) => translate(v)).mkString(", ")

From 9d495cb70b6530adb7608ff2f633e686eb778a59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 27 Jan 2017 20:33:26 +0100
Subject: [PATCH 024/230] CalcIntType -> long, because int might lead to loss
 of data in some calculations.

---
 .../scala/io/kaitai/struct/translators/JavaTranslator.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 3b988f6dd..178f205e5 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -62,7 +62,7 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   }
 
   override def doSubscript(container: expr, idx: expr): String =
-    s"${translate(container)}.get((int) ${translate(idx)})"
+    s"${translate(container)}.get(Long.valueOf(${translate(idx)}).intValue())"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
 

From 6a539d9094ff8d513c6c909ad5515cae7a5cae93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 27 Jan 2017 20:54:40 +0100
Subject: [PATCH 025/230] doSubscript improved to not generate
 Long.valueOf(...) if only an int lietral is provided. which should be a
 regular use case. Long.valueOf(...) shoudl have been optimized in that case
 for low indices as well, but this way we don't need to care and don't add
 extra source to the result.

---
 .../kaitai/struct/translators/JavaTranslator.scala   | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 178f205e5..7e48d823e 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -61,8 +61,16 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     }
   }
 
-  override def doSubscript(container: expr, idx: expr): String =
-    s"${translate(container)}.get(Long.valueOf(${translate(idx)}).intValue())"
+  override def doSubscript(container: expr, idx: expr): String = {
+    val idxStr = translate(idx);
+    val contStr = translate(container);
+    val idxArgStr = idx match {
+      case Ast.expr.IntNum(_) => idxStr
+      case _ => s"Long.valueOf(${idxStr}).intValue()"
+    }
+
+    s"${contStr}.get(${idxArgStr})"
+  }
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
 

From 1d686dafb3a68d9ea4493d2bdca7e5b5042062bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Sun, 29 Jan 2017 17:44:57 +0100
Subject: [PATCH 026/230] CalcIntType=long make a test fail where a literal
 array with large ints is created: new ArrayList<Long>(Arrays.asList(0, 1,
 100500)) doIntLiteral doesn't know that a Long is required and the va.lues
 itself don't require one,. so doIntLiteral keeps providing the numeric
 literals without "L" suffix, which results in "int"s by default in Java and
 an array of int can't be casted to a required array of long.

---
 .../struct/translators/TranslatorSpec.scala   |  2 +-
 .../struct/translators/JavaTranslator.scala   | 27 +++++++++++++++++--
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index d72a07826..4bedf7cf7 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -171,7 +171,7 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks {
     // Arrays
     full("[0, 1, 100500]", CalcIntType, ArrayType(CalcIntType), Map[LanguageCompilerStatic, String](
       CSharpCompiler -> "new List<int> { 0, 1, 100500 }",
-      JavaCompiler -> "new ArrayList<Integer>(Arrays.asList(0, 1, 100500))",
+      JavaCompiler -> "new ArrayList<Long>(Arrays.asList(0L, 1L, 100500L))",
       JavaScriptCompiler -> "[0, 1, 100500]",
       PerlCompiler -> "(0, 1, 100500)",
       PHPCompiler -> "[0, 1, 100500]",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 7e48d823e..f01db9f8e 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -3,7 +3,7 @@ package io.kaitai.struct.translators
 import io.kaitai.struct.Utils
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.exprlang.Ast._
-import io.kaitai.struct.exprlang.DataType.{BaseType, IntType}
+import io.kaitai.struct.exprlang.DataType.{BaseType, CalcIntType, IntType}
 import io.kaitai.struct.languages.JavaCompiler
 
 class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
@@ -14,9 +14,32 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     s"${literal}${suffix}"
   }
 
+  /**
+   * Wrapper for {@link #doIntLiteral(BigInt)} if {@code CalcIntType} is known to be needed.
+   * <p>
+   * {@link #doIntLiteral(BigInt)} doesn't work for statements like {@code new ArrayList<Long>(Arrays.asList(0, 1, 100500))}
+   * because it doesn't know that a {@code long} is always needed, even if the value of the number
+   * wouldn't need it. Java by default assumes {@code int} for numeric literals and would create an
+   * array with a different type than required.
+   * </p>
+   */
+  def doIntLiteralCalcIntType(n: BigInt): String = {
+    val literal = doIntLiteral(n)
+    val isLong = JavaCompiler.kaitaiType2JavaTypePrim(CalcIntType) == "long"
+    val suffixNeeded = if (isLong && !literal.endsWith("L")) true else false
+    val suffix = if (suffixNeeded) "L" else ""
+
+    s"${literal}${suffix}"
+  }
+
   override def doArrayLiteral(t: BaseType, value: Seq[expr]): String = {
     val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t)
-    val commaStr = value.map((v) => translate(v)).mkString(", ")
+    val values = t match {
+      case CalcIntType => value.map((v) => doIntLiteralCalcIntType(v match { case Ast.expr.IntNum(n) => n }))
+      case _ => value.map((v) => translate(v))
+    }
+    val commaStr = values.mkString(", ")
+
     s"new ArrayList<$javaType>(Arrays.asList($commaStr))"
   }
 

From 90731379fc8943842214aaa794a9f35b188b23e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 30 Jan 2017 08:25:41 +0100
Subject: [PATCH 027/230] 
 https://www.codacy.com/app/greycat-na-kor/kaitai_struct_compiler/pullRequest?prid=494963

---
 .../scala/io/kaitai/struct/translators/JavaTranslator.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index f01db9f8e..e247c2cc8 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -26,7 +26,7 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   def doIntLiteralCalcIntType(n: BigInt): String = {
     val literal = doIntLiteral(n)
     val isLong = JavaCompiler.kaitaiType2JavaTypePrim(CalcIntType) == "long"
-    val suffixNeeded = if (isLong && !literal.endsWith("L")) true else false
+    val suffixNeeded = isLong && !literal.endsWith("L")
     val suffix = if (suffixNeeded) "L" else ""
 
     s"${literal}${suffix}"

From c8e2ef6652c6bc04c70f11d029167399908389dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 30 Jan 2017 15:49:12 +0100
Subject: [PATCH 028/230] match may not be exhaustive.

---
 .../scala/io/kaitai/struct/translators/JavaTranslator.scala  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index e247c2cc8..9f11b0048 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -35,7 +35,10 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   override def doArrayLiteral(t: BaseType, value: Seq[expr]): String = {
     val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t)
     val values = t match {
-      case CalcIntType => value.map((v) => doIntLiteralCalcIntType(v match { case Ast.expr.IntNum(n) => n }))
+      case CalcIntType => value.map((v) => v match {
+        case Ast.expr.IntNum(n) => doIntLiteralCalcIntType(n)
+        case _ => throw new UnsupportedOperationException("CalcIntType should only be used for numbers.")
+      })
       case _ => value.map((v) => translate(v))
     }
     val commaStr = values.mkString(", ")

From b4646584a3cd9346c12faf0af39e127f178291ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 7 Feb 2017 17:15:09 +0100
Subject: [PATCH 029/230] Enhanced ArrayType with the size property, so that
 one can check how many records already have been parsed or how many at all
 etc. Depends on when this property is accessed by whom.

---
 .../io/kaitai/struct/translators/TranslatorSpec.scala | 11 +++++++++++
 .../io/kaitai/struct/translators/BaseTranslator.scala |  3 +++
 .../kaitai/struct/translators/CSharpTranslator.scala  |  2 ++
 .../io/kaitai/struct/translators/CppTranslator.scala  |  2 ++
 .../struct/translators/JavaScriptTranslator.scala     |  2 ++
 .../io/kaitai/struct/translators/JavaTranslator.scala |  2 ++
 .../io/kaitai/struct/translators/PHPTranslator.scala  |  2 ++
 .../io/kaitai/struct/translators/PerlTranslator.scala |  2 ++
 .../kaitai/struct/translators/PythonTranslator.scala  |  2 ++
 .../io/kaitai/struct/translators/RubyTranslator.scala |  2 ++
 10 files changed, 30 insertions(+)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 4bedf7cf7..f2e214fc6 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -237,6 +237,17 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks {
       RubyCompiler -> "a.last"
     )),
 
+    full("a.size", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String](
+        CppCompiler -> "a()->size()",
+        CSharpCompiler -> "A.Count",
+        JavaCompiler -> "a().size()",
+        JavaScriptCompiler -> "this.a.length",
+        PHPCompiler -> "count(a)",
+        PerlCompiler -> "scalar($self->a())",
+        PythonCompiler -> "len(self.a)",
+        RubyCompiler -> "a.length"
+      )),
+
     // Strings
     full("\"str\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
       CppCompiler -> "std::string(\"str\")",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 9fb4ea4ff..8de149dff 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -91,6 +91,7 @@ abstract class BaseTranslator(val provider: TypeProvider) {
             attr.name match {
               case "first" => arrayFirst(value)
               case "last" => arrayLast(value)
+              case "size" => arraySize(value)
             }
           case KaitaiStreamType =>
             attr.name match {
@@ -220,6 +221,7 @@ abstract class BaseTranslator(val provider: TypeProvider) {
 
   def arrayFirst(a: Ast.expr): String
   def arrayLast(a: Ast.expr): String
+  def arraySize(a: Ast.expr): String
 
   def kaitaiStreamSize(value: Ast.expr): String = userTypeField(value, "size")
   def kaitaiStreamEof(value: Ast.expr): String = userTypeField(value, "is_eof")
@@ -334,6 +336,7 @@ abstract class BaseTranslator(val provider: TypeProvider) {
           case ArrayType(inType) =>
             attr.name match {
               case "first" | "last" => inType
+              case "size" => CalcIntType
               case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
             }
           case KaitaiStreamType =>
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
index 9abe0c822..4e98bae5a 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
@@ -72,4 +72,6 @@ class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider)
     val v = translate(a)
     s"$v[$v.Length - 1]"
   }
+  override def arraySize(a: expr): String =
+    s"${translate(a)}.Count"
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index ec29c084c..4603e1d5f 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -80,4 +80,6 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     s"${translate(a)}->front()"
   override def arrayLast(a: expr): String =
     s"${translate(a)}->back()"
+  override def arraySize(a: expr): String =
+    s"${translate(a)}->size()"
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
index fc1573d48..b7d6fa690 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
@@ -62,6 +62,8 @@ class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provid
     val v = translate(a)
     s"$v[$v.length - 1]"
   }
+  override def arraySize(a: expr): String =
+    s"${translate(a)}.length"
 
   override def kaitaiStreamEof(value: Ast.expr): String =
     s"${translate(value)}.isEof()"
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 9f11b0048..0d569a70d 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -116,4 +116,6 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     val v = translate(a)
     s"$v.get($v.size() - 1)"
   }
+  override def arraySize(a: expr): String =
+    s"${translate(a)}.size()"
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
index d89c3fd87..a6ae96368 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
@@ -75,4 +75,6 @@ class PHPTranslator(provider: TypeProvider, lang: PHPCompiler) extends BaseTrans
     val v = translate(a)
     s"$v[$v.length - 1]"
   }
+  override def arraySize(a: expr): String =
+    s"count(${translate(a)})"
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
index 2873152b6..2e16718a4 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
@@ -100,6 +100,8 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     s"${translate(a)}[0]"
   override def arrayLast(a: expr): String =
     s"${translate(a)}[-1]"
+  override def arraySize(a: expr): String =
+    s"scalar(${translate(a)})"
 
   override def kaitaiStreamSize(value: Ast.expr): String =
     s"${translate(value)}->size()"
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
index 498aeec13..0ed0d620b 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
@@ -78,6 +78,8 @@ class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider)
     s"${translate(a)}[0]"
   override def arrayLast(a: Ast.expr): String =
     s"${translate(a)}[-1]"
+  override def arraySize(a: Ast.expr): String =
+    s"len(${translate(a)})"
 
   override def kaitaiStreamSize(value: Ast.expr): String =
     s"${translate(value)}.size()"
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
index d63c04d0f..07a243af1 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -39,6 +39,8 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     s"${translate(a)}.first"
   override def arrayLast(a: Ast.expr): String =
     s"${translate(a)}.last"
+  override def arraySize(a: Ast.expr): String =
+    s"${translate(a)}.length"
 
   override def kaitaiStreamEof(value: Ast.expr): String =
     s"${translate(value)}.eof?"

From d97b224bedd544dac3db806538128023ed59ff00 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 7 Feb 2017 19:55:18 +0100
Subject: [PATCH 030/230] Added the operation "reverse" on strings. Only C++ is
 not working currently.

---
 .../io/kaitai/struct/translators/TranslatorSpec.scala | 11 +++++++++++
 .../io/kaitai/struct/translators/BaseTranslator.scala |  3 +++
 .../kaitai/struct/translators/CSharpTranslator.scala  |  6 ++++++
 .../io/kaitai/struct/translators/CppTranslator.scala  |  7 +++++++
 .../struct/translators/JavaScriptTranslator.scala     |  4 ++++
 .../io/kaitai/struct/translators/JavaTranslator.scala |  2 ++
 .../io/kaitai/struct/translators/PHPTranslator.scala  |  3 ++-
 .../io/kaitai/struct/translators/PerlTranslator.scala |  2 ++
 .../kaitai/struct/translators/PythonTranslator.scala  |  2 ++
 .../io/kaitai/struct/translators/RubyTranslator.scala |  2 ++
 10 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index f2e214fc6..297e1035a 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -300,6 +300,17 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks {
       RubyCompiler -> "\"str\".size"
     )),
 
+    full("\"str\".reverse", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
+        CppCompiler -> "std::reverse(...)",
+        CSharpCompiler -> "new string(Array.Reverse(\"str\".ToCharArray()))",
+        JavaCompiler -> "new StringBuilder(\"str\").reverse().toString()",
+        JavaScriptCompiler -> "Array.from(\"str\").reverse().join('')",
+        PerlCompiler -> "scalar(reverse(\"str\"))",
+        PHPCompiler -> "strrev(\"str\")",
+        PythonCompiler -> "u\"str\"[::-1]",
+        RubyCompiler -> "\"str\".reverse"
+      )),
+
     full("\"12345\".to_i", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String](
       CppCompiler -> "std::stoi(std::string(\"12345\"))",
       CSharpCompiler -> "Convert.ToInt64(\"12345\", 10)",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 8de149dff..fa65d63d4 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -81,6 +81,7 @@ abstract class BaseTranslator(val provider: TypeProvider) {
           case _: StrType =>
             attr.name match {
               case "length" => strLength(value)
+              case "reverse" => strReverse(value)
               case "to_i" => strToInt(value, Ast.expr.IntNum(10))
             }
           case _: IntType =>
@@ -217,6 +218,7 @@ abstract class BaseTranslator(val provider: TypeProvider) {
   def strToInt(s: Ast.expr, base: Ast.expr): String
   def intToStr(i: Ast.expr, base: Ast.expr): String
   def strLength(s: Ast.expr): String
+  def strReverse(s: Ast.expr): String
   def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String
 
   def arrayFirst(a: Ast.expr): String
@@ -325,6 +327,7 @@ abstract class BaseTranslator(val provider: TypeProvider) {
           case _: StrType =>
             attr.name match {
               case "length" => CalcIntType
+              case "reverse" => CalcStrType
               case "to_i" => CalcIntType
               case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
             }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
index 4e98bae5a..0d5519f54 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
@@ -63,6 +63,12 @@ class CSharpTranslator(provider: TypeProvider) extends BaseTranslator(provider)
     s"Convert.ToString(${translate(i)}, ${translate(base)})"
   override def strLength(s: expr): String =
     s"${translate(s)}.Length"
+
+  // FIXME: This is not fully Unicode aware, but might be better than nothing.
+  // http://stackoverflow.com/a/228060/2055163
+  override def strReverse(s: expr): String =
+    s"new string(Array.Reverse(${translate(s)}.ToCharArray()))"
+
   override def strSubstring(s: expr, from: expr, to: expr): String =
     s"${translate(s)}.Substring(${translate(from)}, ${translate(to)} - ${translate(from)})"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 4603e1d5f..6e636ead7 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -73,6 +73,13 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   }
   override def strLength(s: expr): String =
     s"${translate(s)}.length()"
+
+  // TODO std::reverse seems the proper thing to do, but needs a local variable to work properly and
+  // the additional header <algorithm>. Maybe there's a better solution?
+  override def strReverse(s: expr): String =
+    //s"std::reverse(${translate(s)}.begin(), ${translate(s)}.end());"
+    throw new RuntimeException("Reversing strings is not implemented yet in C++.")
+
   override def strSubstring(s: expr, from: expr, to: expr): String =
     s"${translate(s)}.substr(${translate(from)}, (${translate(to)}) - (${translate(from)}))"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
index b7d6fa690..1f50c2a76 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
@@ -53,6 +53,10 @@ class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provid
   override def strLength(s: expr): String =
     s"${translate(s)}.length"
 
+  // http://stackoverflow.com/a/36525647/2055163
+  override def strReverse(s: expr): String =
+    s"Array.from(${translate(s)}).reverse().join('')"
+
   override def strSubstring(s: expr, from: expr, to: expr): String =
     s"${translate(s)}.substring(${translate(from)}, ${translate(to)})"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 0d569a70d..f805d5415 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -107,6 +107,8 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     s"Long.toString(${translate(i)}, ${translate(base)})"
   override def strLength(s: expr): String =
     s"${translate(s)}.length()"
+  override def strReverse(s: expr): String =
+    s"new StringBuilder(${translate(s)}).reverse().toString()"
   override def strSubstring(s: expr, from: expr, to: expr): String =
     s"${translate(s)}.substring(${translate(from)}, ${translate(to)})"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
index a6ae96368..cadda4d5b 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
@@ -65,7 +65,8 @@ class PHPTranslator(provider: TypeProvider, lang: PHPCompiler) extends BaseTrans
 
   override def strLength(s: expr): String =
     s"strlen(${translate(s)})"
-
+  override def strReverse(s: expr): String =
+    s"strrev(${translate(s)})"
   override def strSubstring(s: expr, from: expr, to: expr): String =
     s"${translate(s)}.substring(${translate(from)}, ${translate(to)})"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
index 2e16718a4..e60ffa909 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
@@ -93,6 +93,8 @@ class PerlTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   }
   override def strLength(value: Ast.expr): String =
     s"length(${translate(value)})"
+  override def strReverse(value: Ast.expr): String =
+    s"scalar(reverse(${translate(value)}))"
   override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String =
     s"${translate(s)}[${translate(from)}:${translate(to)}]"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
index 0ed0d620b..fb25288aa 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
@@ -71,6 +71,8 @@ class PythonTranslator(provider: TypeProvider) extends BaseTranslator(provider)
   }
   override def strLength(value: Ast.expr): String =
     s"len(${translate(value)})"
+  override def strReverse(value: Ast.expr): String =
+    s"${translate(value)}[::-1]"
   override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String =
     s"${translate(s)}[${translate(from)}:${translate(to)}]"
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
index 07a243af1..b3b63ec6e 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -32,6 +32,8 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   }
   override def strLength(s: Ast.expr): String =
     s"${translate(s)}.size"
+  override def strReverse(s: Ast.expr): String =
+    s"${translate(s)}.reverse"
   override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String =
     s"${translate(s)}[${translate(from)}, (${translate(to)} - 1)]"
 

From 40e887872bf18bfd5c5860643180335f9a0fd272 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 7 Feb 2017 20:33:55 +0100
Subject: [PATCH 031/230] kaitai::kstream::reverse for implementing reverse in
 cpp for strings.

---
 .../io/kaitai/struct/translators/TranslatorSpec.scala      | 2 +-
 .../scala/io/kaitai/struct/translators/CppTranslator.scala | 7 +------
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 297e1035a..183bcf412 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -301,7 +301,7 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks {
     )),
 
     full("\"str\".reverse", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
-        CppCompiler -> "std::reverse(...)",
+        CppCompiler -> "kaitai::kstream::reverse(std::string(\"str\"))",
         CSharpCompiler -> "new string(Array.Reverse(\"str\".ToCharArray()))",
         JavaCompiler -> "new StringBuilder(\"str\").reverse().toString()",
         JavaScriptCompiler -> "Array.from(\"str\").reverse().join('')",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 6e636ead7..1825d8775 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -73,13 +73,8 @@ class CppTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   }
   override def strLength(s: expr): String =
     s"${translate(s)}.length()"
-
-  // TODO std::reverse seems the proper thing to do, but needs a local variable to work properly and
-  // the additional header <algorithm>. Maybe there's a better solution?
   override def strReverse(s: expr): String =
-    //s"std::reverse(${translate(s)}.begin(), ${translate(s)}.end());"
-    throw new RuntimeException("Reversing strings is not implemented yet in C++.")
-
+    s"${CppCompiler.kstreamName}::reverse(${translate(s)})"
   override def strSubstring(s: expr, from: expr, to: expr): String =
     s"${translate(s)}.substr(${translate(from)}, (${translate(to)}) - (${translate(from)}))"
 

From c004df7d4ee18ef0b24a0266053f688b61e02018 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 16 Feb 2017 10:05:55 +0100
Subject: [PATCH 032/230] My changes for _parent_vs_value_inst won't be
 accepted, so I'm testing in another branch the merge without my changes.
 Those lead to a merge conflict witht he solution of GreyCat currently.


From d8b2aa3dcd9978b58368dd939577107ee3437ee4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?= <tschoening@am-soft.de>
Date: Thu, 16 Feb 2017 10:17:15 +0100
Subject: [PATCH 033/230] Revert "Merge branch '_parent_vs_value_inst' into
 libs_java_3rd_usage"

This reverts commit 252af0f6b642728ac6af4e4939decd476a98fb2a, reversing
changes made to f87e3541bf61f8d622c9256d49317cbe34f5d2e4.
---
 .../io/kaitai/struct/ClassTypeProvider.scala  | 11 +++------
 .../io/kaitai/struct/TypeProcessor.scala      | 24 ++-----------------
 2 files changed, 5 insertions(+), 30 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
index 8df1d15ab..861cd2df4 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
@@ -5,8 +5,7 @@ import io.kaitai.struct.format._
 import io.kaitai.struct.translators.{TypeMismatchError, TypeProvider, TypeUndecidedError}
 
 class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider {
-  var nowClass: ClassSpec = topClass
-  var possibleParentClass: Option[ClassSpec] = None
+  var nowClass = topClass
 
   var _currentIteratorType: Option[BaseType] = None
   var _currentSwitchType: Option[BaseType] = None
@@ -22,13 +21,9 @@ class ClassTypeProvider(topClass: ClassSpec) extends TypeProvider {
       case "_root" =>
         makeUserType(topClass)
       case "_parent" =>
-        var parent = inClass.parentClass
-        if ((parent == UnknownClassSpec) && (possibleParentClass != None))
-          parent = possibleParentClass.get
-        if (parent == UnknownClassSpec)
+        if (inClass.parentClass == UnknownClassSpec)
           throw new RuntimeException(s"Unable to derive _parent type in ${inClass.name.mkString("::")}")
-
-        makeUserType(parent)
+        makeUserType(inClass.parentClass)
       case "_io" =>
         KaitaiStreamType
       case "_" =>
diff --git a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
index 67e19b45e..e84e8e34f 100644
--- a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
+++ b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
@@ -5,28 +5,17 @@ import io.kaitai.struct.format._
 import io.kaitai.struct.translators.{BaseTranslator, RubyTranslator, TypeUndecidedError}
 
 import scala.collection.mutable.ListBuffer
-import scala.collection.mutable.Map
 
 object TypeProcessor {
-  // We need to record the parent types of resolved user types at their usage, which we know about
-  // on resolving user types, to be able to properly resolve "_parent" in value instances to solve
-  // some chicken&egg problem in the current implementation.
-  //
-  // The key is a string by purpose currently, because on deriving value there may be some opaque
-  // types and for some reasion their ClassSpec couldn't be queried as a key in the map without
-  // running into UnsupportedOperationException of AbstractIdentifier.toString. This doesn't happen
-  // if we provide a string using ClassSpec on our own.
-  val parentsByChild = Map[String, ClassSpec]()
-
   def processTypes(topClass: ClassSpec): Unit = {
     // Set top class name from meta
     topClass.name = List(topClass.meta.get.id.get)
-    topClass.parentClass = GenericStructClassSpec
 
     markupClassNames(topClass)
     resolveUserTypes(topClass)
     deriveValueTypes(topClass)
     markupParentTypes(topClass)
+    topClass.parentClass = GenericStructClassSpec
   }
 
   // ==================================================================
@@ -64,8 +53,6 @@ object TypeProcessor {
     var hasChanged = false
 
     provider.nowClass = curClass
-    provider.possibleParentClass = parentsByChild.get(curClass.name.mkString("::"))
-
     curClass.instances.foreach {
       case (instName, inst) =>
         inst match {
@@ -152,15 +139,8 @@ object TypeProcessor {
       case None =>
         // Type definition not found - generate special "opaque placeholder" ClassSpec
         Some(ClassSpec.opaquePlaceholder(typeName))
-      case Some(child) => {
-        // We might be called multiple times, but only have two situations: It's either always the
-        // same context or in case of different/incompatible ones there's some chance that the last
-        // call is the proper one, so we simply overwrite mappings. Depending on the target language
-        // in case of multiple incompatible usages compiler errors occur or such and things can be
-        // fixed as needed.
-        parentsByChild(child.name.mkString("::")) = curClass
+      case Some(x) =>
         res
-      }
     }
   }
 

From 9116b8425cc798785931e4760f7df0f80f768e7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 24 Mar 2017 10:25:56 +0100
Subject: [PATCH 034/230] Resolved a former merge conflict wrongly, overlooked
 some changes which lead to compiler errors.

---
 .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 251812e90..22570285f 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -528,7 +528,7 @@ object JavaCompiler extends LanguageCompilerStatic
 
       case BitsType(_) => "long"
 
-      case BooleanType => "boolean"
+      case _: BooleanType => "boolean"
       case CalcIntType => "long"
       case CalcFloatType => "double"
 
@@ -572,7 +572,7 @@ object JavaCompiler extends LanguageCompilerStatic
 
       case BitsType(_) => "Long"
 
-      case BooleanType => "Boolean"
+      case _: BooleanType => "Boolean"
       case CalcIntType => "Long"
       case CalcFloatType => "Double"
 

From 5a4d398e0448ca18f77a9453d7090c9df2e649c2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 31 Mar 2017 17:22:09 +0200
Subject: [PATCH 035/230] needRaw didn't properly work for SwitchType, which
 might concists of different types for all cases. It should be sufficient to
 test if one of those cases fullfills needRaw to get the overall result for
 needRaw for the whole switch.

---
 .../languages/components/EveryReadIsExpression.scala     | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
index 9ad3f56cd..9888ce5cc 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
@@ -193,10 +193,19 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage
   def needRaw(dataType: DataType): Boolean = {
     dataType match {
       case t: UserTypeFromBytes => true
+      case t: SwitchType => needRaw(t)
       case _ => false
     }
   }
 
+  def needRaw(switchType: SwitchType): Boolean = {
+    val byCase = switchType.cases.mapValues(v => needRaw(v))
+    val byBool = byCase.groupBy(pair => pair._2)
+    val retVal = byBool.contains(true)
+
+    retVal
+  }
+
   def attrSwitchTypeParse(id: Identifier, on: Ast.expr, cases: Map[Ast.expr, DataType], io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec): Unit = {
     switchStart(id, on)
 

From 50d97b34b824bfe4ed91e1184400f3bfe453aa72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 31 Mar 2017 17:45:03 +0200
Subject: [PATCH 036/230] 
 https://www.codacy.com/app/greycat-na-kor/kaitai_struct_compiler/pullRequest?prid=587517

---
 .../struct/languages/components/EveryReadIsExpression.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
index 9888ce5cc..c4016eb62 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
@@ -200,7 +200,7 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage
 
   def needRaw(switchType: SwitchType): Boolean = {
     val byCase = switchType.cases.mapValues(v => needRaw(v))
-    val byBool = byCase.groupBy(pair => pair._2)
+    val byBool = byCase.groupBy({ case (k, v) => v })
     val retVal = byBool.contains(true)
 
     retVal

From bc9d8df491f44a57940f7f77409be69ad8ec6148 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 3 Apr 2017 15:45:15 +0200
Subject: [PATCH 037/230] Normalize path handling, so that / only is used, Make
 sinteraction with ksv easier and fixes escape problems with \.

---
 shared/src/main/scala/io/kaitai/struct/JSON.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/JSON.scala b/shared/src/main/scala/io/kaitai/struct/JSON.scala
index 4e2be6070..9d9940976 100644
--- a/shared/src/main/scala/io/kaitai/struct/JSON.scala
+++ b/shared/src/main/scala/io/kaitai/struct/JSON.scala
@@ -25,7 +25,7 @@ object JSON {
 
   // FIXME: do proper string handling
   def stringToJson(str: String): String =
-    "\"%s\"".format(str)
+    "\"%s\"".format(str.replaceAll("\\\\", "/"))
 
   def listToJson(obj: List[_]): String =
     "[" + obj.map((x) => stringify(x)).mkString(",") + "]"

From c7db43eb89feed980d2c5ed0490759110e72dede Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 4 Apr 2017 09:50:14 +0200
Subject: [PATCH 038/230] @GreyCat decided for a different solution and I will
 switch to that.

---
 shared/src/main/scala/io/kaitai/struct/JSON.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/JSON.scala b/shared/src/main/scala/io/kaitai/struct/JSON.scala
index 9d9940976..4e2be6070 100644
--- a/shared/src/main/scala/io/kaitai/struct/JSON.scala
+++ b/shared/src/main/scala/io/kaitai/struct/JSON.scala
@@ -25,7 +25,7 @@ object JSON {
 
   // FIXME: do proper string handling
   def stringToJson(str: String): String =
-    "\"%s\"".format(str.replaceAll("\\\\", "/"))
+    "\"%s\"".format(str)
 
   def listToJson(obj: List[_]): String =
     "[" + obj.map((x) => stringify(x)).mkString(",") + "]"

From eea181c503bc74c054f0c01acd72b7cdbf44b05d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 11 Apr 2017 17:48:32 +0200
Subject: [PATCH 039/230] 
 https://github.com/kaitai-io/kaitai_struct_compiler/issues/82

---
 .../struct/languages/components/EveryReadIsExpression.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
index c4016eb62..ed94b01fe 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
@@ -184,8 +184,8 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage
         case _ =>
           val tempVarName = s"_t_${idToStr(id)}"
           handleAssignmentTempVar(dataType, tempVarName, expr)
-          handleAssignment(id, tempVarName, rep, false)
           userTypeDebugRead(tempVarName)
+          handleAssignment(id, tempVarName, rep, false)
       }
     }
   }

From ddce146dc25c205776410b2b5dcbf655e8f03224 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?= <tschoening@am-soft.de>
Date: Wed, 14 Feb 2018 17:15:30 +0100
Subject: [PATCH 040/230] Revert "needRaw didn't properly work for
 SwitchType[...]" because a different solution has been implemented in the
 meanwhile and my changes lead to unnecessary merge conflicts.

https://github.com/kaitai-io/kaitai_struct/issues/205
https://github.com/kaitai-io/kaitai_struct_compiler/commit/a7f1167717c15ba3ce1480d67c4cab2eacfad196#diff-3b9e2ea7df1c77e6702738c198f9cd35R84

https://github.com/kaitai-io/kaitai_struct_compiler/issues/78
https://github.com/kaitai-io/kaitai_struct_compiler/pull/79

This reverts commit 5a4d398e0448ca18f77a9453d7090c9df2e649c2.
---
 .../languages/components/EveryReadIsExpression.scala     | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
index ed94b01fe..cefb05751 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
@@ -193,19 +193,10 @@ trait EveryReadIsExpression extends LanguageCompiler with ObjectOrientedLanguage
   def needRaw(dataType: DataType): Boolean = {
     dataType match {
       case t: UserTypeFromBytes => true
-      case t: SwitchType => needRaw(t)
       case _ => false
     }
   }
 
-  def needRaw(switchType: SwitchType): Boolean = {
-    val byCase = switchType.cases.mapValues(v => needRaw(v))
-    val byBool = byCase.groupBy({ case (k, v) => v })
-    val retVal = byBool.contains(true)
-
-    retVal
-  }
-
   def attrSwitchTypeParse(id: Identifier, on: Ast.expr, cases: Map[Ast.expr, DataType], io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec): Unit = {
     switchStart(id, on)
 

From d766bfae7150d5d3731029baf1752e1a8871a516 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?= <tschoening@am-soft.de>
Date: Wed, 14 Feb 2018 17:40:21 +0100
Subject: [PATCH 041/230] * Revert "CalcIntType -> long", because my changes
 for the compiler don't seem to get merged easily. It's a complicated topic
 and I don't want to have more merge conflicts than needed, so are removing
 those changes from the branch I'm working on for now. The branch for the
 changes is kept for future merges, so I won't loose anything.

This reverts commit 71365ce1b37950ecec2705bb7868f8a2260cc190.
This reverts commit 03904283dcf4f8adb8581a8ac6731f08fe6de6e8.
This reverts commit c9c984783e16702139581463a8e89da54b074d48.
This reverts commit b7686fa81b9c2992afd18676bb311d6e8020db29.
This reverts commit d785266ea0d13f14b03b4ea40fcd0cacfb7585ab.
This reverts commit f816bfaedff9eec77d94c085705b1ad240cf0a10.
---
 .../struct/translators/TranslatorSpec.scala   |  2 +-
 .../struct/languages/JavaCompiler.scala       |  8 ++--
 .../struct/translators/JavaTranslator.scala   | 48 ++-----------------
 3 files changed, 8 insertions(+), 50 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 31765aa87..872fde5d7 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -184,7 +184,7 @@ class TranslatorSpec extends FunSuite with TableDrivenPropertyChecks {
     // Arrays
     full("[0, 1, 100500]", CalcIntType, ArrayType(CalcIntType), Map[LanguageCompilerStatic, String](
       CSharpCompiler -> "new List<int> { 0, 1, 100500 }",
-      JavaCompiler -> "new ArrayList<Long>(Arrays.asList(0L, 1L, 100500L))",
+      JavaCompiler -> "new ArrayList<Integer>(Arrays.asList(0, 1, 100500))",
       JavaScriptCompiler -> "[0, 1, 100500]",
       PerlCompiler -> "(0, 1, 100500)",
       PHPCompiler -> "[0, 1, 100500]",
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 486768bbf..6ff7ad6c3 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -270,8 +270,8 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>(Long.valueOf(${expression(repeatExpr)}).intValue());")
-    out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}(Long.valueOf(${expression(repeatExpr)}).intValue());")
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>((int) (${expression(repeatExpr)}));")
+    out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}((int) (${expression(repeatExpr)}));")
     out.puts(s"for (int i = 0; i < ${expression(repeatExpr)}; i++) {")
     out.inc
   }
@@ -530,7 +530,7 @@ object JavaCompiler extends LanguageCompilerStatic
       case BitsType(_) => "long"
 
       case _: BooleanType => "boolean"
-      case CalcIntType => "long"
+      case CalcIntType => "int"
       case CalcFloatType => "double"
 
       case _: StrType => "String"
@@ -574,7 +574,7 @@ object JavaCompiler extends LanguageCompilerStatic
       case BitsType(_) => "Long"
 
       case _: BooleanType => "Boolean"
-      case CalcIntType => "Long"
+      case CalcIntType => "Integer"
       case CalcFloatType => "Double"
 
       case _: StrType => "String"
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 074606847..a268e2b6e 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -9,42 +9,9 @@ import io.kaitai.struct.format.Identifier
 import io.kaitai.struct.languages.JavaCompiler
 
 class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
-  override def doIntLiteral(n: BigInt): String = {
-    val literal = n.toString
-    val suffix = if (n > Int.MaxValue) "L" else ""
-
-    s"${literal}${suffix}"
-  }
-
-  /**
-   * Wrapper for {@link #doIntLiteral(BigInt)} if {@code CalcIntType} is known to be needed.
-   * <p>
-   * {@link #doIntLiteral(BigInt)} doesn't work for statements like {@code new ArrayList<Long>(Arrays.asList(0, 1, 100500))}
-   * because it doesn't know that a {@code long} is always needed, even if the value of the number
-   * wouldn't need it. Java by default assumes {@code int} for numeric literals and would create an
-   * array with a different type than required.
-   * </p>
-   */
-  def doIntLiteralCalcIntType(n: BigInt): String = {
-    val literal = doIntLiteral(n)
-    val isLong = JavaCompiler.kaitaiType2JavaTypePrim(CalcIntType) == "long"
-    val suffixNeeded = isLong && !literal.endsWith("L")
-    val suffix = if (suffixNeeded) "L" else ""
-
-    s"${literal}${suffix}"
-  }
-
   override def doArrayLiteral(t: DataType, value: Seq[expr]): String = {
     val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t)
-    val values = t match {
-      case CalcIntType => value.map((v) => v match {
-        case Ast.expr.IntNum(n) => doIntLiteralCalcIntType(n)
-        case _ => throw new UnsupportedOperationException("CalcIntType should only be used for numbers.")
-      })
-      case _ => value.map((v) => translate(v))
-    }
-    val commaStr = values.mkString(", ")
-
+    val commaStr = value.map((v) => translate(v)).mkString(", ")
     s"new ArrayList<$javaType>(Arrays.asList($commaStr))"
   }
 
@@ -101,17 +68,8 @@ class JavaTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     }
   }
 
-  override def doSubscript(container: expr, idx: expr): String = {
-    val idxStr = translate(idx);
-    val contStr = translate(container);
-    val idxArgStr = idx match {
-      case Ast.expr.IntNum(_) => idxStr
-      case _ => s"Long.valueOf(${idxStr}).intValue()"
-    }
-
-    s"${contStr}.get(${idxArgStr})"
-  }
-
+  override def doSubscript(container: expr, idx: expr): String =
+    s"${translate(container)}.get((int) ${translate(idx)})"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
   override def doCast(value: Ast.expr, typeName: String): String =

From 347e594de5035cc691c8ce76616fbf17b48d47bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 16 Feb 2018 09:53:05 +0100
Subject: [PATCH 042/230] Compiler version 0.8 doesn't build on Windows (10)
 with sbt.version = 1.1.0-RC4

https://github.com/kaitai-io/kaitai_struct/issues/362
---
 project/build.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/build.properties b/project/build.properties
index e98ac44be..210243d0d 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -1 +1 @@
-sbt.version = 1.1.0-RC4
+sbt.version = 1.1.1

From ff6b0461a27dcb9e738c673f86cef574bc666720 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 16 Feb 2018 09:21:27 +0000
Subject: [PATCH 043/230] Upgrade to sbt 1.1.1, trying to fix Windows 10 vs sbt
 incompatibility, as per https://github.com/kaitai-io/kaitai_struct/issues/362

---
 project/build.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/build.properties b/project/build.properties
index e98ac44be..210243d0d 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -1 +1 @@
-sbt.version = 1.1.0-RC4
+sbt.version = 1.1.1

From 97b4ef19b8487d7e56b9b13e5237586d7d666a8a Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Feb 2018 01:07:49 +0000
Subject: [PATCH 044/230] Added support for user types in params definitions,
 should help https://github.com/kaitai-io/kaitai_struct/issues/363

---
 .../main/scala/io/kaitai/struct/datatype/DataType.scala   | 1 +
 .../scala/io/kaitai/struct/precompile/ResolveTypes.scala  | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index 78e111043..0919fe70a 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -348,6 +348,7 @@ object DataType {
         case "struct" => KaitaiStructType
         case "io" => KaitaiStreamType
         case "any" => AnyType
+        case _ => UserTypeInstream(classNameToList(dt), None)
       }
     }
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
index 1fe9eb0c6..f659abdc9 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
@@ -18,19 +18,21 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) {
     * @param curClass class to start from, might be top-level class
     */
   def resolveUserTypes(curClass: ClassSpec): Unit = {
-    curClass.seq.foreach((attr) => resolveUserTypeForAttr(curClass, attr))
+    curClass.seq.foreach((attr) => resolveUserTypeForMember(curClass, attr))
 
     curClass.instances.foreach { case (_, instSpec) =>
       instSpec match {
         case pis: ParseInstanceSpec =>
-          resolveUserTypeForAttr(curClass, pis)
+          resolveUserTypeForMember(curClass, pis)
         case _: ValueInstanceSpec =>
           // ignore all other types of instances
       }
     }
+
+    curClass.params.foreach((paramDef) => resolveUserTypeForMember(curClass, paramDef))
   }
 
-  def resolveUserTypeForAttr(curClass: ClassSpec, attr: AttrLikeSpec): Unit =
+  def resolveUserTypeForMember(curClass: ClassSpec, attr: MemberSpec): Unit =
     resolveUserType(curClass, attr.dataType, attr.path)
 
   def resolveUserType(curClass: ClassSpec, dataType: DataType, path: List[String]): Unit = {

From ee644aa5da85fc22a372e7a34522738792f033af Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Feb 2018 10:37:30 +0000
Subject: [PATCH 045/230] Going forward, starting 0.9

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index bf0d21f88..c138295b6 100644
--- a/build.sbt
+++ b/build.sbt
@@ -5,7 +5,7 @@ import sbt.Keys._
 
 resolvers += Resolver.sonatypeRepo("public")
 
-val VERSION = "0.8"
+val VERSION = "0.9-SNAPSHOT"
 val TARGET_LANGS = "C++/STL, C#, Java, JavaScript, Lua, Perl, PHP, Python, Ruby"
 
 lazy val root = project.in(file(".")).

From 1481959432e9d4ad367869793b7f8d3957b3c7fe Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Feb 2018 01:07:49 +0000
Subject: [PATCH 046/230] Added support for user types in params definitions,
 should help https://github.com/kaitai-io/kaitai_struct/issues/363

---
 .../main/scala/io/kaitai/struct/datatype/DataType.scala   | 1 +
 .../scala/io/kaitai/struct/precompile/ResolveTypes.scala  | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index 78e111043..0919fe70a 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -348,6 +348,7 @@ object DataType {
         case "struct" => KaitaiStructType
         case "io" => KaitaiStreamType
         case "any" => AnyType
+        case _ => UserTypeInstream(classNameToList(dt), None)
       }
     }
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
index 1fe9eb0c6..f659abdc9 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
@@ -18,19 +18,21 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) {
     * @param curClass class to start from, might be top-level class
     */
   def resolveUserTypes(curClass: ClassSpec): Unit = {
-    curClass.seq.foreach((attr) => resolveUserTypeForAttr(curClass, attr))
+    curClass.seq.foreach((attr) => resolveUserTypeForMember(curClass, attr))
 
     curClass.instances.foreach { case (_, instSpec) =>
       instSpec match {
         case pis: ParseInstanceSpec =>
-          resolveUserTypeForAttr(curClass, pis)
+          resolveUserTypeForMember(curClass, pis)
         case _: ValueInstanceSpec =>
           // ignore all other types of instances
       }
     }
+
+    curClass.params.foreach((paramDef) => resolveUserTypeForMember(curClass, paramDef))
   }
 
-  def resolveUserTypeForAttr(curClass: ClassSpec, attr: AttrLikeSpec): Unit =
+  def resolveUserTypeForMember(curClass: ClassSpec, attr: MemberSpec): Unit =
     resolveUserType(curClass, attr.dataType, attr.path)
 
   def resolveUserType(curClass: ClassSpec, dataType: DataType, path: List[String]): Unit = {

From dbda9b765e1c57f8e2e5dcfcf7d33dc7b1e8e48e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Feb 2018 10:58:34 +0000
Subject: [PATCH 047/230] JavaTranslator: use clever trick with hex literals to
 squeeze unsigned 64-bit integers

---
 .../io/kaitai/struct/translators/JavaTranslator.scala     | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index e72097bcc..3ae4cd00e 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -10,10 +10,14 @@ import io.kaitai.struct.languages.JavaCompiler
 
 class JavaTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) {
   override def doIntLiteral(n: BigInt): String = {
-    val literal = n.toString
+    val literal = if (n > Long.MaxValue) {
+      "0x" + n.toString(16)
+    } else {
+      n.toString
+    }
     val suffix = if (n > Int.MaxValue) "L" else ""
 
-    s"${literal}${suffix}"
+    s"$literal$suffix"
   }
 
   override def doArrayLiteral(t: DataType, value: Seq[expr]): String = {

From 8217da16c3c3771d807714069f200d0e2a715816 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 26 Feb 2018 12:48:02 +0100
Subject: [PATCH 048/230] Make accessing a foreign type possible for
 declarations and getters in Java.

---
 .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index bde35cd47..c20b1781d 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -765,7 +765,7 @@ object JavaCompiler extends LanguageCompilerStatic
       case KaitaiStreamType => kstreamName
       case KaitaiStructType => kstructName
 
-      case t: UserType => type2class(t.name.last)
+      case t: UserType => types2class(t.name)
       case EnumType(name, _) => types2class(name)
 
       case ArrayType(inType) => s"ArrayList<${kaitaiType2JavaTypeBoxed(inType)}>"

From afd09549d685e4f35d3ff29865b4edd29740dba3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 26 Feb 2018 12:48:02 +0100
Subject: [PATCH 049/230] Make accessing a foreign type possible for
 declarations and getters in Java.

---
 .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index bde35cd47..c20b1781d 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -765,7 +765,7 @@ object JavaCompiler extends LanguageCompilerStatic
       case KaitaiStreamType => kstreamName
       case KaitaiStructType => kstructName
 
-      case t: UserType => type2class(t.name.last)
+      case t: UserType => types2class(t.name)
       case EnumType(name, _) => types2class(name)
 
       case ArrayType(inType) => s"ArrayList<${kaitaiType2JavaTypeBoxed(inType)}>"

From 66aa397193a01caa46b073211403382258eb59e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 26 Feb 2018 14:28:42 +0100
Subject: [PATCH 050/230] realResolveUserType needs to work recursively in top
 level specs.

---
 .../io/kaitai/struct/precompile/ResolveTypes.scala     | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
index f659abdc9..99282b35c 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
@@ -107,7 +107,15 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) {
             } else {
               // Check if top-level specs has this name
               // If there's None => no luck at all
-              specs.get(firstName)
+              val resolvedTop = specs.get(firstName)
+              resolvedTop match {
+                case None => None
+                case Some(classSpec) => restNames match {
+                  case null => resolvedTop
+                  case List() => resolvedTop
+                  case _ => resolveUserType(classSpec, restNames, path)
+                }
+              }
             }
         }
     }

From 058823c68319fce74245ad1901278d181491d3fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Mon, 26 Feb 2018 14:28:42 +0100
Subject: [PATCH 051/230] realResolveUserType needs to work recursively in top
 level specs.

---
 .../io/kaitai/struct/precompile/ResolveTypes.scala     | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
index f659abdc9..99282b35c 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
@@ -107,7 +107,15 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) {
             } else {
               // Check if top-level specs has this name
               // If there's None => no luck at all
-              specs.get(firstName)
+              val resolvedTop = specs.get(firstName)
+              resolvedTop match {
+                case None => None
+                case Some(classSpec) => restNames match {
+                  case null => resolvedTop
+                  case List() => resolvedTop
+                  case _ => resolveUserType(classSpec, restNames, path)
+                }
+              }
             }
         }
     }

From 791b1a1f2dd7b14fb8dd5c95e9ce6f93b14c486b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 27 Feb 2018 09:09:30 +0100
Subject: [PATCH 052/230] "match" is not used for "restNames" for
 "resolvedHere", so don't use it later as well.

---
 .../scala/io/kaitai/struct/precompile/ResolveTypes.scala  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
index 99282b35c..4ee06ecc0 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
@@ -110,10 +110,10 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) {
               val resolvedTop = specs.get(firstName)
               resolvedTop match {
                 case None => None
-                case Some(classSpec) => restNames match {
-                  case null => resolvedTop
-                  case List() => resolvedTop
-                  case _ => resolveUserType(classSpec, restNames, path)
+                case Some(classSpec) => if (restNames.isEmpty) {
+                  resolvedTop
+                } else {
+                  resolveUserType(classSpec, restNames, path)
                 }
               }
             }

From ebbdedf87fe1d93b1cb51c242e1fb78dbc9167d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 27 Feb 2018 09:09:30 +0100
Subject: [PATCH 053/230] "match" is not used for "restNames" for
 "resolvedHere", so don't use it later as well.

---
 .../scala/io/kaitai/struct/precompile/ResolveTypes.scala  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
index 99282b35c..4ee06ecc0 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
@@ -110,10 +110,10 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) {
               val resolvedTop = specs.get(firstName)
               resolvedTop match {
                 case None => None
-                case Some(classSpec) => restNames match {
-                  case null => resolvedTop
-                  case List() => resolvedTop
-                  case _ => resolveUserType(classSpec, restNames, path)
+                case Some(classSpec) => if (restNames.isEmpty) {
+                  resolvedTop
+                } else {
+                  resolveUserType(classSpec, restNames, path)
                 }
               }
             }

From c12fbd85698ca5773e8d8c060556f3422b8ee587 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 28 Feb 2018 01:00:05 +0000
Subject: [PATCH 054/230] Added Bintray publishing lib + utility

---
 lib_bintray.sh        | 65 +++++++++++++++++++++++++++++++++++++++++++
 publish_to_bintray.sh | 21 ++++++++++++++
 2 files changed, 86 insertions(+)
 create mode 100644 lib_bintray.sh
 create mode 100755 publish_to_bintray.sh

diff --git a/lib_bintray.sh b/lib_bintray.sh
new file mode 100644
index 000000000..6d3729ac8
--- /dev/null
+++ b/lib_bintray.sh
@@ -0,0 +1,65 @@
+# Shell library to handle uploads & publishing of artifacts to Bintray
+
+# All functions get their settings from global variables:
+#
+# * Authentication:
+#   * BINTRAY_USER
+#   * BINTRAY_API_KEY - to be passed as secret env variable#
+# * Package ID / upload coordinates:
+#   * BINTRAY_ACCOUNT
+#   * BINTRAY_REPO
+#   * BINTRAY_PACKAGE
+#   * BINTRAY_VERSION
+# * Debian-specific settings:
+#   * BINTRAY_DEB_DISTRIBUTION
+#   * BINTRAY_DEB_ARCH
+#   * BINTRAY_DEB_COMPONENT
+# * Debug options:
+#   * BINTRAY_CURL_ARGS - set to `-vv` for verbose output
+
+##
+# Creates version for a package at Bintray
+bintray_create_version()
+{
+	echo "bintray_create_version(repo=${BINTRAY_REPO}, package=${BINTRAY_PACKAGE}, version=${BINTRAY_VERSION})"
+
+	curl $BINTRAY_CURL_ARGS -f \
+		"-u$BINTRAY_USER:$BINTRAY_API_KEY" \
+		-H "Content-Type: application/json" \
+		-X POST "https://api.bintray.com/packages/${BINTRAY_ACCOUNT}/${BINTRAY_REPO}/${BINTRAY_PACKAGE}/versions" \
+		--data "{ \"name\": \"$BINTRAY_VERSION\", \"release_notes\": \"auto\", \"released\": \"\" }"
+#         --data "{ \"name\": \"$version\", \"release_notes\": \"auto\", \"release_url\": \"$BASE_DESC/$RPM_NAME\", \"released\": \"\" }"
+}
+
+##
+# Uploads deb package to Bintray.
+#
+# Input:
+# $1 = filename to upload
+bintray_upload_deb()
+{
+	local filename="$1"
+
+	echo "bintray_upload_deb(repo=${BINTRAY_REPO}, package=${BINTRAY_PACKAGE}, version=${BINTRAY_VERSION}, filename=${filename})"
+
+	curl $BINTRAY_CURL_ARGS -f \
+		-T "$filename" \
+		"-u$BINTRAY_USER:$BINTRAY_API_KEY" \
+		-H "X-Bintray-Package: $BINTRAY_PACKAGE" \
+		-H "X-Bintray-Version: $BINTRAY_VERSION" \
+		-H "X-Bintray-Debian-Distribution: $BINTRAY_DEB_DISTRIBUTION" \
+		-H "X-Bintray-Debian-Architecture: $BINTRAY_DEB_ARCH" \
+		-H "X-Bintray-Debian-Component: $BINTRAY_DEB_COMPONENT" \
+		https://api.bintray.com/content/$BINTRAY_ACCOUNT/$BINTRAY_REPO/
+}
+
+bintray_publish_version()
+{
+	echo "bintray_publish_version(repo=${BINTRAY_REPO}, package=${BINTRAY_PACKAGE}, version=${BINTRAY_VERSION})"
+
+	curl $BINTRAY_CURL_ARGS -f \
+		"-u$BINTRAY_USER:$BINTRAY_API_KEY" \
+		-H "Content-Type: application/json" \
+		-X POST "https://api.bintray.com/content/$BINTRAY_ACCOUNT/$BINTRAY_REPO/$BINTRAY_PACKAGE/$BINTRAY_VERSION/publish" \
+		--data "{ \"discard\": \"false\" }"
+}
diff --git a/publish_to_bintray.sh b/publish_to_bintray.sh
new file mode 100755
index 000000000..28a557ab6
--- /dev/null
+++ b/publish_to_bintray.sh
@@ -0,0 +1,21 @@
+#!/bin/sh -ef
+
+. ./lib_bintray.sh
+
+# Config
+BINTRAY_USER=greycat
+BINTRAY_ACCOUNT=kaitai-io
+BINTRAY_REPO=debian_unstable
+BINTRAY_PACKAGE=kaitai-struct-compiler
+BINTRAY_VERSION="$KAITAI_STRUCT_VERSION"
+# BINTRAY_API_KEY comes from encrypted variables from web UI
+
+BINTRAY_DEB_DISTRIBUTION=jessie
+BINTRAY_DEB_ARCH=all
+BINTRAY_DEB_COMPONENT=main
+
+#BINTRAY_CURL_ARGS=-v
+
+bintray_create_version
+bintray_upload_deb "jvm/target/kaitai-struct-compiler_${KAITAI_STRUCT_VERSION}_all.deb"
+bintray_publish_version

From 77ff751c75346340c9e13e9ff74fa081b749248f Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 2 Mar 2018 11:30:03 +0000
Subject: [PATCH 055/230] Rewrote README, removing duplicating information,
 replacing it with links

---
 README.md | 111 +++++-------------------------------------------------
 1 file changed, 9 insertions(+), 102 deletions(-)

diff --git a/README.md b/README.md
index 188a05fa6..4cca194e2 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,7 @@
 
 # Kaitai Struct: compiler
 
-This project is an official reference compiler for [Kaitai Struct](https://github.com/kaitai-io/kaitai_struct) project.
+This project is an official reference compiler for [Kaitai Struct](http://kaitai.io) project.
 
 Kaitai Struct is a declarative language used to describe various
 binary data structures, laid out in files or in memory: i.e. binary
@@ -19,109 +19,16 @@ languages. These modules will include the generated code for a parser
 that can read described data structure from a file / stream and give
 access to it in a nice, easy-to-comprehend API.
 
-Please refer to [documentation in Kaitai Struct project](https://github.com/kaitai-io/kaitai_struct)
-for details on `.ksy` files and general usage patterns.
+## Further information
 
-## Trying without install
+If you're looking for information on:
 
-Kaitai Struct compiler can be tried instantly, without any downloads
-and installation, at
-
-http://kaitai.io/repl
-
-Note that this implementation uses the same reference code as in this
-repository and executes totally on a client side, without any queries
-to server backend.
-
-## Downloading and installing
-
-### Linux .deb builds (Debian/Ubuntu)
-
-There is an official .deb repository available. The repository is hosted
-at BinTray and signed with BinTray GPG key (`379CE192D401AB61`), so it's
-necessary to import that key first if your box haven't used any BinTray
-repositories beforehand:
-
-```shell
-echo "deb https://dl.bintray.com/kaitai-io/debian jessie main" | sudo tee /etc/apt/sources.list.d/kaitai.list
-sudo apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv 379CE192D401AB61
-sudo apt-get update
-sudo apt-get install kaitai-struct-compiler
-```
-
-### Windows builds
-
-An official `.msi` installer build is available to download at
-
-https://bintray.com/kaitai-io/universal/kaitai-struct-compiler/_latestVersion
-
-### Universal builds
-
-Basically, everything that can run Java can use so called "universal"
-builds: a .zip file that includes all the required .jar files bundled
-and launcher scripts for UNIX/Windows systems. No installation
-required, one can just unpack and run it. Available at download also at
-
-https://bintray.com/kaitai-io/universal/kaitai-struct-compiler/_latestVersion
-
-### Source code
-
-If you're interested in developing compiler itself, you can check out
-source code in repository:
-
-    git clone https://github.com/kaitai-io/kaitai_struct_compiler
-
-See the [developer documentation](http://doc.kaitai.io/developers.html) for 
-general pointers on how to proceed with the source code then.
-
-## Usage
-
-`kaitai-struct-compiler [options] <file>...`
-
-Alternatively, a symlink `ksc` is provided and can be used everywhere
-just as full name.
-
-Common options:
-
-* `<file>...` — source files (.ksy)
-* `-t <language> | --target <language>` — target languages (`graphviz`, `csharp`,
-    `all`, `perl`, `java`, `go`, `cpp_stl`, `php`, `lua`, `python`, `ruby`, `javascript`
-  * `all` is a special case: it compiles all possible target
-    languages, creating language-specific directories (as per language
-    identifiers) inside output directory, and then creating output
-    module(s) for each language starting from there
-* `-d <directory> | --outdir <directory>` — output directory
-  (filenames will be auto-generated)
-
-Language-specific options:
-
-* `--dot-net-namespace <namespace>` — .NET namespace (C# only, default: Kaitai)
-* `--java-package <package>` — Java package (Java only, default: root package)
-* `--php-namespace <namespace>` — PHP namespace (PHP only, default: root package)
-
-Misc options:
-
-* `--verbose` — verbose output
-* `--help` — display usage information and exit
-* `--version` — output version information and exit
-
-A few examples, given that file `foo.ksy` exists in current directory
-and describes format with ID `foo`:
-
-* `kaitai-struct-compiler -t python foo.ksy` — compile format in
-  `foo.ksy`, write output in current directory to file `foo.py`
-* `kaitai-struct-compiler -t java foo.ksy` — compile format in
-  `foo.ksy`, create "src" subdir in current one and write output in
-  `src/Foo.java`
-* `kaitai-struct-compiler -t java --java-package org.example foo.ksy`
-  — compile format in `foo.ksy`, create "src/org/example" subdir tree
-  in current one and write output in `src/org/example/Foo.java`;
-  resulting file will bear correct Java package clause.
-* `kaitai-struct-compiler -t all -d /tmp/out --java-package org.example foo.ksy`
-  — compile format in `foo.ksy`, creating a hierarchy of files:
-  * `/tmp/out/java/src/org/example/Foo.java`
-  * `/tmp/out/python/foo.py`
-  * `/tmp/out/ruby/foo.rb`
+* Kaitai Struct language itself (`.ksy` files, general usage patterns)
+  — refer to the [user guide](http://doc.kaitai.io/user_guide.html).
+* How to download and install Kaitai Struct — see the
+  [downloads](http://kaitai.io/#download).
+* How to build the compiler, run the test suite, and join the
+  development — see the [developer memo](http://doc.kaitai.io/developers.html).
 
 ## Licensing
 

From 09d1ce795e9c4572c855a74d9a47f80ef2b95893 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 6 Mar 2018 11:25:56 +0000
Subject: [PATCH 056/230] Added java8-runtime-headless to deb packages dep

---
 build.sbt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/build.sbt b/build.sbt
index c138295b6..08c9a9ae1 100644
--- a/build.sbt
+++ b/build.sbt
@@ -128,6 +128,8 @@ lazy val compiler = crossProject.in(file(".")).
     // https://github.com/sbt/sbt-native-packager/issues/1067
     debianNativeBuildOptions in Debian := Seq("-Zgzip", "-z3"),
 
+    debianPackageDependencies := Seq("java8-runtime-headless"),
+
     packageSummary in Linux := s"compiler to generate binary data parsers in $TARGET_LANGS",
     packageSummary in Windows := "Kaitai Struct compiler",
     packageDescription in Linux :=

From 56a8f38fb5fe1c70aba7d788d1be9352d2880280 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 8 Mar 2018 16:55:01 +0100
Subject: [PATCH 057/230] doArrayLiteral used classes without importing the
 necessary packages.

---
 .../scala/io/kaitai/struct/translators/JavaTranslator.scala    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 3ae4cd00e..947c14e28 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -23,6 +23,9 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
   override def doArrayLiteral(t: DataType, value: Seq[expr]): String = {
     val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t)
     val commaStr = value.map((v) => translate(v)).mkString(", ")
+
+    importList.add("java.util.ArrayList")
+    importList.add("java.util.Arrays")
     s"new ArrayList<$javaType>(Arrays.asList($commaStr))"
   }
 

From c12b265aefcb1b92e4736d1914e632842c4b45b9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 8 Mar 2018 16:55:01 +0100
Subject: [PATCH 058/230] doArrayLiteral used classes without importing the
 necessary packages.

---
 .../scala/io/kaitai/struct/translators/JavaTranslator.scala    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 3ae4cd00e..947c14e28 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -23,6 +23,9 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
   override def doArrayLiteral(t: DataType, value: Seq[expr]): String = {
     val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t)
     val commaStr = value.map((v) => translate(v)).mkString(", ")
+
+    importList.add("java.util.ArrayList")
+    importList.add("java.util.Arrays")
     s"new ArrayList<$javaType>(Arrays.asList($commaStr))"
   }
 

From af54072fbe9c2981dfd600a3edbc2fd4b479b2d5 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 16 Mar 2018 02:33:46 +0000
Subject: [PATCH 059/230] RubyCompiler: fixed deeply nested usertype classes
 invocation

---
 .../main/scala/io/kaitai/struct/languages/RubyCompiler.scala  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index 1e588698a..914b7bfa2 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -361,7 +361,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           }
           s", $parent, @_root$addEndian"
         }
-        s"${type2class(t.name.last)}.new($io$addArgs$addParams)"
+        s"${types2class(t.name)}.new($io$addArgs$addParams)"
     }
   }
 
@@ -452,6 +452,8 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def publicMemberName(id: Identifier): String = idToStr(id)
 
   override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}"
+
+  def types2class(names: List[String]) = names.map(x => type2class(x)).mkString("::")
 }
 
 object RubyCompiler extends LanguageCompilerStatic

From 458b6410ed55aed493c349795b1ff9bd9083c81c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 16 Mar 2018 11:14:33 +0000
Subject: [PATCH 060/230] JavaScriptCompiler: fix nested_types3 user type
 construction

---
 .../io/kaitai/struct/languages/JavaScriptCompiler.scala      | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
index f3d5fb3f7..7361f27b2 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
@@ -388,7 +388,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           case _ => ""
         }
         val addParams = Utils.join(t.args.map((a) => translator.translate(a)), ", ", ", ", "")
-        s"new ${type2class(t.name.last)}($io, $parent, $root$addEndian$addParams)"
+        s"new ${types2class(t.name)}($io, $parent, $root$addEndian$addParams)"
     }
   }
 
@@ -597,6 +597,5 @@ object JavaScriptCompiler extends LanguageCompilerStatic
   // FIXME: probably KaitaiStruct will emerge some day in JavaScript runtime, but for now it is unused
   override def kstructName: String = ???
 
-  def types2class(types: List[String]): String =
-    types.map(JavaScriptCompiler.type2class).mkString(".")
+  def types2class(types: List[String]): String = types.map(type2class).mkString(".")
 }

From 3abace306d40645a99e9ecf9b369185769e42c20 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 16 Mar 2018 11:15:01 +0000
Subject: [PATCH 061/230] RubyCompiler: style fix for brevity

---
 .../main/scala/io/kaitai/struct/languages/RubyCompiler.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index 914b7bfa2..9138981e1 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -453,7 +453,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}"
 
-  def types2class(names: List[String]) = names.map(x => type2class(x)).mkString("::")
+  def types2class(names: List[String]) = names.map(type2class).mkString("::")
 }
 
 object RubyCompiler extends LanguageCompilerStatic

From 97fa50e2f837adc643dbb2a675493a94f1a7bded Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 17 Mar 2018 03:46:06 +0000
Subject: [PATCH 062/230] * Expressions: added distinct entity "typeId", which
 supports   `a::b::c` or `::a::b::c` style type naming * Allowed to use that
 entity in `.as<type_id>` type casts,   i.e. `.as<a::b::c>` * Fixed type
 resolving to support that in run-time * Fixed all the APIs to support more
 than single identifier for type   naming

---
 .../struct/exprlang/ExpressionsSpec.scala     | 32 ++++++++++++++++---
 .../struct/translators/TranslatorSpec.scala   |  8 ++---
 .../io/kaitai/struct/ClassTypeProvider.scala  | 27 ++++++++++++----
 .../scala/io/kaitai/struct/exprlang/Ast.scala |  3 +-
 .../kaitai/struct/exprlang/Expressions.scala  |  6 +++-
 .../struct/languages/CSharpCompiler.scala     |  5 ++-
 .../kaitai/struct/languages/CppCompiler.scala |  8 +++++
 .../struct/translators/BaseTranslator.scala   |  4 +--
 .../struct/translators/CSharpTranslator.scala |  4 +--
 .../struct/translators/CppTranslator.scala    |  4 +--
 .../struct/translators/JavaTranslator.scala   |  6 ++--
 .../struct/translators/TypeDetector.scala     |  2 +-
 .../struct/translators/TypeProvider.scala     |  3 +-
 13 files changed, 84 insertions(+), 28 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
index 5e0fe54e3..63bad3f4f 100644
--- a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
@@ -202,23 +202,45 @@ class ExpressionsSpec extends FunSpec {
 
     // Casts
     it("parses 123.as<u4>") {
-      Expressions.parse("123.as<u4>") should be (CastToType(IntNum(123),identifier("u4")))
+      Expressions.parse("123.as<u4>") should be (
+        CastToType(IntNum(123), typeId(false, Seq("u4")))
+      )
     }
 
     it("parses (123).as<u4>") {
-      Expressions.parse("(123).as<u4>") should be (CastToType(IntNum(123),identifier("u4")))
+      Expressions.parse("(123).as<u4>") should be (
+        CastToType(IntNum(123), typeId(false, Seq("u4")))
+      )
     }
 
     it("parses \"str\".as<x>") {
-      Expressions.parse("\"str\".as<x>") should be (CastToType(Str("str"),identifier("x")))
+      Expressions.parse("\"str\".as<x>") should be (
+        CastToType(Str("str"), typeId(false, Seq("x")))
+      )
     }
 
     it("parses foo.as<x>") {
-      Expressions.parse("foo.as<x>") should be (CastToType(Name(identifier("foo")),identifier("x")))
+      Expressions.parse("foo.as<x>") should be (
+        CastToType(Name(identifier("foo")), typeId(false, Seq("x")))
+      )
     }
 
     it("parses foo.as < x  >  ") {
-      Expressions.parse("foo.as < x  >  ") should be (CastToType(Name(identifier("foo")),identifier("x")))
+      Expressions.parse("foo.as < x  >  ") should be (
+        CastToType(Name(identifier("foo")), typeId(false, Seq("x")))
+      )
+    }
+
+    it("parses foo.as<bar::baz>") {
+      Expressions.parse("foo.as<bar::baz>") should be (
+        CastToType(Name(identifier("foo")), typeId(false, Seq("bar", "baz")))
+      )
+    }
+
+    it("parses foo.as<::bar::baz>") {
+      Expressions.parse("foo.as<::bar::baz>") should be (
+        CastToType(Name(identifier("foo")), typeId(true, Seq("bar", "baz")))
+      )
     }
 
     it("parses foo.as") {
diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 7789addba..3d888e92e 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -472,7 +472,7 @@ class TranslatorSpec extends FunSuite {
     override def resolveEnum(enumName: String) =
       throw new NotImplementedError
 
-    override def resolveType(typeName: String): DataType =
+    override def resolveType(typeName: Ast.typeId): DataType =
       throw new NotImplementedError
 
     override def isLazy(attrName: String): Boolean = false
@@ -500,9 +500,9 @@ class TranslatorSpec extends FunSuite {
       }
     }
 
-    override def resolveType(typeName: String): DataType = {
-      typeName match {
-        case "top_class" | "block" | "innerblock" => userType(typeName)
+    override def resolveType(typeName: Ast.typeId): DataType = {
+      typeName.names.head match {
+        case "top_class" | "block" | "innerblock" => userType(typeName.names.head)
       }
     }
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
index 11eda9e07..b7de515cb 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
@@ -2,6 +2,7 @@ package io.kaitai.struct
 
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
 import io.kaitai.struct.precompile.{EnumNotFoundError, FieldNotFoundError, TypeNotFoundError, TypeUndecidedError}
 import io.kaitai.struct.translators.TypeProvider
@@ -84,22 +85,36 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends
     }
   }
 
-  override def resolveType(typeName: String): DataType = resolveType(nowClass, typeName)
+  override def resolveType(typeName: Ast.typeId): DataType =
+    resolveType(if (typeName.absolute) topClass else nowClass, typeName.names)
 
-  def resolveType(inClass: ClassSpec, typeName: String): DataType = {
+  def resolveType(inClass: ClassSpec, typeName: Seq[String]): DataType =
+    makeUserType(resolveClassSpec(inClass, typeName))
+
+  def resolveClassSpec(inClass: ClassSpec, typeName: Seq[String]): ClassSpec = {
+    val headTypeName :: restTypesNames = typeName
+    val nextClass = resolveClassSpec(inClass, headTypeName)
+    if (restTypesNames.isEmpty) {
+      nextClass
+    } else {
+      resolveClassSpec(nextClass, restTypesNames)
+    }
+  }
+
+  def resolveClassSpec(inClass: ClassSpec, typeName: String): ClassSpec = {
     if (inClass.name.last == typeName)
-      return makeUserType(inClass)
+      return inClass
 
     inClass.types.get(typeName) match {
       case Some(spec) =>
-        makeUserType(spec)
+        spec
       case None =>
         // let's try upper levels of hierarchy
         inClass.upClass match {
-          case Some(upClass) => resolveType(upClass, typeName)
+          case Some(upClass) => resolveClassSpec(upClass, typeName)
           case None =>
             classSpecs.get(typeName) match {
-              case Some(spec) => makeUserType(spec)
+              case Some(spec) => spec
               case None =>
                 throw new TypeNotFoundError(typeName, nowClass)
             }
diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
index ff23decd6..ad27bd840 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
@@ -21,6 +21,7 @@ package io.kaitai.struct.exprlang
   */
 object Ast {
   case class identifier(name: String)
+  case class typeId(absolute: Boolean, names: Seq[String])
 
   // BoolOp() can use left & right?
   sealed trait expr
@@ -40,7 +41,7 @@ object Ast {
     case class EnumById(enumName: identifier, id: expr) extends expr
 
     case class Attribute(value: expr, attr: identifier) extends expr
-    case class CastToType(value: expr, typeName: identifier) extends expr
+    case class CastToType(value: expr, typeName: typeId) extends expr
     case class Subscript(value: expr, idx: expr) extends expr
     case class Name(id: identifier) extends expr
     case class List(elts: Seq[expr]) extends expr
diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
index 1e0d6f1ea..ef887999d 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
@@ -27,6 +27,10 @@ import fastparse.StringReprOps
 object Expressions {
 
   val NAME: P[Ast.identifier] = Lexical.identifier
+  val TYPE_NAME: P[Ast.typeId] = P("::".!.? ~ NAME.rep(1, "::")).map {
+    case (first, names: Seq[Ast.identifier]) =>
+      Ast.typeId(first.nonEmpty, names.map((el) => el.name))
+  }
   val INT_NUMBER = Lexical.integer
   val FLOAT_NUMBER = Lexical.floatnumber
   val STRING: P[String] = Lexical.stringliteral
@@ -130,7 +134,7 @@ object Expressions {
   val trailer: P[Ast.expr => Ast.expr] = {
     val call = P("(" ~ arglist ~ ")").map{ case (args) => (lhs: Ast.expr) => Ast.expr.Call(lhs, args)}
     val slice = P("[" ~ test ~ "]").map{ case (args) => (lhs: Ast.expr) => Ast.expr.Subscript(lhs, args)}
-    val cast = P( "." ~ "as" ~ "<" ~ NAME ~ ">" ).map(
+    val cast = P( "." ~ "as" ~ "<" ~ TYPE_NAME ~ ">" ).map(
       typeName => (lhs: Ast.expr) => Ast.expr.CastToType(lhs, typeName)
     )
     val attr = P("." ~ NAME).map(id => (lhs: Ast.expr) => Ast.expr.Attribute(lhs, id))
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
index 55fcd971a..50e0fa128 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
@@ -625,7 +625,10 @@ object CSharpCompiler extends LanguageCompilerStatic
     }
   }
 
-  def types2class(names: List[String]) = names.map(x => type2class(x)).mkString(".")
+  def types2class(typeName: Ast.typeId): String =
+    // FIXME: handle absolute
+    types2class(typeName.names)
+  def types2class(names: Iterable[String]) = names.map(type2class).mkString(".")
 
   override def kstructName = "KaitaiStruct"
   override def kstreamName = "KaitaiStream"
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 3ebf25e96..989e7bea9 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -869,6 +869,14 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
   override def kstructName = "kaitai::kstruct"
   override def kstreamName = "kaitai::kstream"
 
+  def types2class(typeName: Ast.typeId) = {
+    typeName.names.mkString(
+      if (typeName.absolute) "::" else "",
+      "::",
+      ""
+    )
+  }
+
   def types2class(components: List[String]) = {
     components.map {
       case "kaitai_struct" => "kaitai::kstruct"
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 2c2c808cb..2bd37ceb3 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -135,13 +135,13 @@ abstract class BaseTranslator(val provider: TypeProvider)
             doArrayLiteral(t, values)
         }
       case Ast.expr.CastToType(value, typeName) =>
-        doCast(value, typeName.name)
+        doCast(value, typeName)
     }
   }
 
   def doSubscript(container: Ast.expr, idx: Ast.expr): String
   def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String
-  def doCast(value: Ast.expr, typeName: String): String = translate(value)
+  def doCast(value: Ast.expr, typeName: Ast.typeId): String = translate(value)
 
   def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = "[" + value.map((v) => translate(v)).mkString(", ") + "]"
   def doByteArrayLiteral(arr: Seq[Byte]): String = "[" + arr.map(_ & 0xff).mkString(", ") + "]"
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
index 26a030b89..db3ab9657 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
@@ -81,8 +81,8 @@ class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends B
     s"${translate(container)}[${translate(idx)}]"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
-  override def doCast(value: Ast.expr, typeName: String): String =
-    s"((${Utils.upperCamelCase(typeName)}) (${translate(value)}))"
+  override def doCast(value: Ast.expr, typeName: Ast.typeId): String =
+    s"((${CSharpCompiler.types2class(typeName)}) (${translate(value)}))"
 
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String = {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 2ee150df3..87aa800cc 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -95,8 +95,8 @@ class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends B
     s"${translate(container)}->at(${translate(idx)})"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"((${translate(condition)}) ? (${translate(ifTrue)}) : (${translate(ifFalse)}))"
-  override def doCast(value: Ast.expr, typeName: String): String =
-    s"static_cast<${CppCompiler.types2class(List(typeName))}*>(${translate(value)})"
+  override def doCast(value: Ast.expr, typeName: Ast.typeId): String =
+    s"static_cast<${CppCompiler.types2class(typeName)}*>(${translate(value)})"
 
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String = {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 947c14e28..c67e25a76 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -90,8 +90,10 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
     s"${translate(container)}.get((int) ${translate(idx)})"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
-  override def doCast(value: Ast.expr, typeName: String): String =
-    s"((${Utils.upperCamelCase(typeName)}) (${translate(value)}))"
+  override def doCast(value: Ast.expr, typeName: Ast.typeId): String = {
+    // FIXME: assuming relative type name
+    s"((${JavaCompiler.types2class(typeName.names.toList)}) (${translate(value)}))"
+  }
 
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index 89be684ba..ba19f4479 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -176,7 +176,7 @@ class TypeDetector(provider: TypeProvider) {
           case t => ArrayType(t)
         }
       case Ast.expr.CastToType(value, typeName) =>
-        provider.resolveType(typeName.name)
+        provider.resolveType(typeName)
     }
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala
index 74f9207d5..119c33cc0 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala
@@ -1,6 +1,7 @@
 package io.kaitai.struct.translators
 
 import io.kaitai.struct.datatype.DataType
+import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.{ClassSpec, EnumSpec}
 
 /**
@@ -13,7 +14,7 @@ trait TypeProvider {
   def determineType(attrName: String): DataType
   def determineType(inClass: ClassSpec, attrName: String): DataType
   def resolveEnum(enumName: String): EnumSpec
-  def resolveType(typeName: String): DataType
+  def resolveType(typeName: Ast.typeId): DataType
   def isLazy(attrName: String): Boolean
   def isLazy(inClass: ClassSpec, attrName: String): Boolean
 }

From 6a22ebbace821ccd50f7f93aa59726a35866314b Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 17 Mar 2018 11:02:37 +0000
Subject: [PATCH 063/230] TranslatorSpec: added multi-name cast test

---
 .../struct/translators/TranslatorSpec.scala   | 61 +++++++++++++++----
 1 file changed, 50 insertions(+), 11 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 3d888e92e..1d33000ee 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -130,7 +130,7 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "foo_str"
   ))
 
-  full("foo_block", userType("block"), userType("block"), Map[LanguageCompilerStatic, String](
+  full("foo_block", userType(List("block")), userType(List("block")), Map[LanguageCompilerStatic, String](
     CppCompiler -> "foo_block()",
     CSharpCompiler -> "FooBlock",
     JavaCompiler -> "fooBlock()",
@@ -166,7 +166,7 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "foo.inner.baz"
   ))
 
-  full("_root.foo", userType("block"), userType("block"), Map[LanguageCompilerStatic, String](
+  full("_root.foo", userType(List("top_class", "block")), userType(List("top_class", "block")), Map[LanguageCompilerStatic, String](
     CppCompiler -> "_root()->foo()",
     CSharpCompiler -> "M_Root.Foo",
     JavaCompiler -> "_root.foo()",
@@ -425,6 +425,18 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "other.bar"
   ))
 
+  full("other.as<block::innerblock>.baz", FooBarProvider, CalcIntType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "static_cast<block::innerblock*>(other())->baz()",
+    CSharpCompiler -> "((Block.Innerblock) (Other)).Baz",
+    JavaCompiler -> "((Block.Innerblock) (other())).baz()",
+    JavaScriptCompiler -> "this.other.baz",
+    LuaCompiler -> "self.other.baz",
+    PerlCompiler -> "$self->other()->baz()",
+    PHPCompiler -> "$this->other()->baz()",
+    PythonCompiler -> "self.other.baz",
+    RubyCompiler -> "other.baz"
+  ))
+
   def runTest(src: String, tp: TypeProvider, expType: DataType, expOut: ResultMap) {
     var eo: Option[Ast.expr] = None
     test(s"_expr:$src") {
@@ -485,30 +497,57 @@ class TranslatorSpec extends FunSuite {
     override def determineType(inClass: ClassSpec, name: String): DataType = t
   }
 
+  /**
+    * Emulates the following system of types:
+    *
+    * <pre>
+    *   meta:
+    *     id: top_class
+    *   types:
+    *     block:
+    *       seq:
+    *         - id: bar
+    *           type: str
+    *         - id: inner
+    *           type: innerblock
+    *       types:
+    *         innerblock:
+    *           instances:
+    *             baz:
+    *               value: 123
+    * </pre>
+    */
   case object FooBarProvider extends FakeTypeProvider {
     override def determineType(name: String): DataType = {
       name match {
-        case "foo" => userType("block")
+        case "foo" => userType(List("top_class", "block"))
       }
     }
 
     override def determineType(inClass: ClassSpec, name: String): DataType = {
-      (inClass.name, name) match {
-        case (List("block"), "bar") => CalcStrType
-        case (List("block"), "inner") => userType("innerblock")
-        case (List("innerblock"), "baz") => CalcIntType
+      (inClass.name.last, name) match {
+        case ("block", "bar") => CalcStrType
+        case ("block", "inner") => userType(List("top_class", "block", "innerblock"))
+        case ("innerblock", "baz") => CalcIntType
       }
     }
 
     override def resolveType(typeName: Ast.typeId): DataType = {
-      typeName.names.head match {
-        case "top_class" | "block" | "innerblock" => userType(typeName.names.head)
+      typeName.names match {
+        case Seq("top_class") =>
+          userType(List("top_class"))
+        case Seq("block") |
+             Seq("top_class", "block") =>
+          userType(List("top_class", "block"))
+        case Seq("innerblock") |
+             Seq("block", "innerblock") |
+             Seq("top_class", "block", "innerblock") =>
+          userType(List("top_class", "block", "innerblock"))
       }
     }
   }
 
-  def userType(name: String) = {
-    val lname = List(name)
+  def userType(lname: List[String]) = {
     val cs = ClassSpec.opaquePlaceholder(lname)
     val ut = UserTypeInstream(lname, None)
     ut.classSpec = Some(cs)

From 95e45dc1a5fe33996a2f9802643f1df5a3194286 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 17 Mar 2018 11:04:15 +0000
Subject: [PATCH 064/230] ClassTypeProvider: list comprehension actually only
 works on lists

---
 shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
index b7de515cb..ce63c7e38 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
@@ -92,7 +92,7 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends
     makeUserType(resolveClassSpec(inClass, typeName))
 
   def resolveClassSpec(inClass: ClassSpec, typeName: Seq[String]): ClassSpec = {
-    val headTypeName :: restTypesNames = typeName
+    val headTypeName :: restTypesNames = typeName.toList
     val nextClass = resolveClassSpec(inClass, headTypeName)
     if (restTypesNames.isEmpty) {
       nextClass

From 5920ced54db5cedbaffa6fddf970dcbd56db83cb Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 18 Mar 2018 10:05:39 +0000
Subject: [PATCH 065/230] CppCompiler: fix typex2class

---
 .../io/kaitai/struct/languages/CppCompiler.scala   | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 989e7bea9..1e85b57e9 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -857,7 +857,7 @@ class CppCompiler(
     }
   }
 
-  def type2class(name: String) = name + "_t"
+  override def type2class(className: String): String = CppCompiler.type2class(className)
 }
 
 object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
@@ -870,17 +870,15 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
   override def kstreamName = "kaitai::kstream"
 
   def types2class(typeName: Ast.typeId) = {
-    typeName.names.mkString(
+    typeName.names.map(type2class).mkString(
       if (typeName.absolute) "::" else "",
       "::",
       ""
     )
   }
 
-  def types2class(components: List[String]) = {
-    components.map {
-      case "kaitai_struct" => "kaitai::kstruct"
-      case s => s + "_t"
-    }.mkString("::")
-  }
+  def types2class(components: List[String]) =
+    components.map(type2class).mkString("::")
+
+  def type2class(name: String) = name + "_t"
 }

From 2c99fe37e0078d9af25df5d5458bc1e5782659ce Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 18 Mar 2018 10:06:01 +0000
Subject: [PATCH 066/230] TranslatorSpec: fix c++ for nested class casting

---
 .../scala/io/kaitai/struct/translators/TranslatorSpec.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 1d33000ee..1f301bbb3 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -426,7 +426,7 @@ class TranslatorSpec extends FunSuite {
   ))
 
   full("other.as<block::innerblock>.baz", FooBarProvider, CalcIntType, Map[LanguageCompilerStatic, String](
-    CppCompiler -> "static_cast<block::innerblock*>(other())->baz()",
+    CppCompiler -> "static_cast<block_t::innerblock_t*>(other())->baz()",
     CSharpCompiler -> "((Block.Innerblock) (Other)).Baz",
     JavaCompiler -> "((Block.Innerblock) (other())).baz()",
     JavaScriptCompiler -> "this.other.baz",

From bc9b96a9d92e1ecc07d52c90376b91cac821d3b6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 20 Mar 2018 16:28:00 +0100
Subject: [PATCH 067/230] Trim expressions to parse or user type invocations
 with attributes can't start with spaces. Seems like an unnecessary
 restriction and prevents proper column based indentation of long attribute
 names.

---
 .../src/main/scala/io/kaitai/struct/exprlang/Expressions.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
index 1e0d6f1ea..50224417d 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
@@ -171,7 +171,7 @@ object Expressions {
   def parseList(src: String): Seq[Ast.expr] = realParse(src, topExprList)
 
   private def realParse[T](src: String, parser: P[T]): T = {
-    val r = parser.parse(src)
+    val r = parser.parse(src.trim)
     r match {
       case Parsed.Success(value, _) => value
       case f: Parsed.Failure =>

From 2095e2a7dab16f113e1e2366aeadbd34d3948678 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Tue, 20 Mar 2018 16:28:00 +0100
Subject: [PATCH 068/230] Trim expressions to parse or user type invocations
 with attributes can't start with spaces. Seems like an unnecessary
 restriction and prevents proper column based indentation of long attribute
 names.

---
 .../src/main/scala/io/kaitai/struct/exprlang/Expressions.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
index ef887999d..f42194df2 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
@@ -175,7 +175,7 @@ object Expressions {
   def parseList(src: String): Seq[Ast.expr] = realParse(src, topExprList)
 
   private def realParse[T](src: String, parser: P[T]): T = {
-    val r = parser.parse(src)
+    val r = parser.parse(src.trim)
     r match {
       case Parsed.Success(value, _) => value
       case f: Parsed.Failure =>

From 09117d637a1dc8b5318128bff5d69609c5bae2b7 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Mar 2018 18:46:28 +0000
Subject: [PATCH 069/230] Added deeply-nested enum ExpressionSpecs + adjusted
 generated types

---
 .../struct/exprlang/ExpressionsSpec.scala     | 31 +++++++++++++++++++
 .../kaitai/struct/GraphvizClassCompiler.scala |  2 +-
 .../scala/io/kaitai/struct/exprlang/Ast.scala |  4 ++-
 .../kaitai/struct/languages/GoCompiler.scala  |  4 +--
 .../struct/languages/JavaCompiler.scala       |  4 +--
 .../struct/translators/BaseTranslator.scala   |  2 +-
 .../struct/translators/TypeDetector.scala     |  2 +-
 7 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
index 63bad3f4f..92a508a3d 100644
--- a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
@@ -131,10 +131,41 @@ class ExpressionsSpec extends FunSpec {
       Expressions.parse("~(7+3)") should be (UnaryOp(Invert, BinOp(IntNum(7), Add, IntNum(3))))
     }
 
+    // Enums
     it("parses port::http") {
       Expressions.parse("port::http") should be (EnumByLabel(identifier("port"), identifier("http")))
     }
 
+    it("parses some_type::port::http") {
+      Expressions.parse("some_type::port::http") should be (
+        EnumByLabel(
+          identifier("port"),
+          identifier("http"),
+          typeId(absolute = false, Seq("some_type"))
+        )
+      )
+    }
+
+    it("parses parent_type::child_type::port::http") {
+      Expressions.parse("parent_type::child_type::port::http") should be (
+        EnumByLabel(
+          identifier("port"),
+          identifier("http"),
+          typeId(absolute = false, Seq("parent_type", "child_type"))
+        )
+      )
+    }
+
+    it("parses ::parent_type::child_type::port::http") {
+      Expressions.parse("::parent_type::child_type::port::http") should be (
+        EnumByLabel(
+          identifier("port"),
+          identifier("http"),
+          typeId(absolute = true, Seq("parent_type", "child_type"))
+        )
+      )
+    }
+
     it("parses port::http.to_i + 8000 == 8080") {
       Expressions.parse("port::http.to_i + 8000 == 8080") should be (
         Compare(
diff --git a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
index 03d8ef1ce..ff172c7e4 100644
--- a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
@@ -307,7 +307,7 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
       //      case expr.Call(func, args) =>
       case expr.IntNum(_) | expr.FloatNum(_) | expr.Str(_) | expr.Bool(_) =>
         List()
-      case expr.EnumByLabel(enumName, label) =>
+      case _: expr.EnumByLabel =>
         List()
       case expr.EnumById(enumName, id) =>
         affectedVars(id)
diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
index ad27bd840..a2f78d237 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
@@ -23,6 +23,8 @@ object Ast {
   case class identifier(name: String)
   case class typeId(absolute: Boolean, names: Seq[String])
 
+  val EmptyTypeId = typeId(false, Seq())
+
   // BoolOp() can use left & right?
   sealed trait expr
   object expr{
@@ -37,7 +39,7 @@ object Ast {
     case class FloatNum(n: BigDecimal) extends expr
     case class Str(s: String) extends expr
     case class Bool(n: Boolean) extends expr
-    case class EnumByLabel(enumName: identifier, label: identifier) extends expr
+    case class EnumByLabel(enumName: identifier, label: identifier, inType: typeId = EmptyTypeId) extends expr
     case class EnumById(enumName: identifier, id: expr) extends expr
 
     case class Attribute(value: expr, attr: identifier) extends expr
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index 430ff940b..5e6aa6a3e 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -300,10 +300,10 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     // Java is very specific about what can be used as "condition" in "case
     // condition:".
     val condStr = condition match {
-      case Ast.expr.EnumByLabel(enumName, enumVal) =>
+      case enumByLabel: Ast.expr.EnumByLabel =>
         // If switch is over a enum, only literal enum values are supported,
         // and they must be written as "MEMBER", not "SomeEnum.MEMBER".
-        value2Const(enumVal.name)
+        value2Const(enumByLabel.label.name)
       case _ =>
         expression(condition)
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index c20b1781d..25247b83a 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -530,10 +530,10 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       // Java is very specific about what can be used as "condition" in "case
       // condition:".
       val condStr = condition match {
-        case Ast.expr.EnumByLabel(_, enumVal) =>
+        case enumByLabel: Ast.expr.EnumByLabel =>
           // If switch is over a enum, only literal enum values are supported,
           // and they must be written as "MEMBER", not "SomeEnum.MEMBER".
-          value2Const(enumVal.name)
+          value2Const(enumByLabel.label.name)
         case _ =>
           expression(condition)
       }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 2bd37ceb3..31fab129c 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -55,7 +55,7 @@ abstract class BaseTranslator(val provider: TypeProvider)
       case Ast.expr.EnumById(enumType, id) =>
         val enumSpec = provider.resolveEnum(enumType.name)
         doEnumById(enumSpec.name, translate(id))
-      case Ast.expr.EnumByLabel(enumType, label) =>
+      case Ast.expr.EnumByLabel(enumType, label, inType) =>
         val enumSpec = provider.resolveEnum(enumType.name)
         doEnumByLabel(enumSpec.name, label.name)
       case Ast.expr.Name(name: Ast.identifier) =>
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index ba19f4479..cc8c8fb1b 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -49,7 +49,7 @@ class TypeDetector(provider: TypeProvider) {
       case Ast.expr.FloatNum(_) => CalcFloatType
       case Ast.expr.Str(_) => CalcStrType
       case Ast.expr.Bool(_) => CalcBooleanType
-      case Ast.expr.EnumByLabel(enumType, _) =>
+      case Ast.expr.EnumByLabel(enumType, _, inType) =>
         val t = EnumType(List(enumType.name), CalcIntType)
         t.enumSpec = Some(provider.resolveEnum(enumType.name))
         t

From 10dae643ac8d11debaa19736c1b4ec1a2a5e441d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Mar 2018 19:29:02 +0000
Subject: [PATCH 070/230] FixedEndian is actually sealed class

---
 .../src/main/scala/io/kaitai/struct/datatype/Endianness.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala b/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala
index 9db938c3b..4606d88e9 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/Endianness.scala
@@ -6,7 +6,7 @@ import io.kaitai.struct.format.{ParseUtils, YAMLParseException}
 
 sealed trait Endianness
 
-abstract class FixedEndian extends Endianness {
+sealed abstract class FixedEndian extends Endianness {
   def toSuffix: String
 }
 case object LittleEndian extends FixedEndian {

From fb0e001e4c62a53316790a9e145c3bff0d1fe234 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Mar 2018 19:30:13 +0000
Subject: [PATCH 071/230] Started ConstructClassCompiler

---
 .../struct/ConstructClassCompiler.scala       | 80 +++++++++++++++++++
 .../main/scala/io/kaitai/struct/Main.scala    |  2 +
 .../components/LanguageCompilerStatic.scala   |  1 +
 3 files changed, 83 insertions(+)
 create mode 100644 shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
new file mode 100644
index 000000000..869ab5642
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -0,0 +1,80 @@
+package io.kaitai.struct
+
+import io.kaitai.struct.datatype._
+import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.format._
+import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic}
+import io.kaitai.struct.translators.PythonTranslator
+
+class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends AbstractCompiler {
+  val out = new StringLanguageOutputWriter(indent)
+  val importList = new ImportList
+
+  val provider = new ClassTypeProvider(classSpecs, topClass)
+  val translator = new PythonTranslator(provider, importList)
+
+  override def compile: CompileLog.SpecSuccess = {
+    out.puts("from construct import *")
+    out.puts("from construct.lib import *")
+    out.puts
+
+    compileClass(topClass)
+
+    out.puts
+    out.puts(s"_schema = ${type2class(topClass)}")
+
+    CompileLog.SpecSuccess(
+      "",
+      List(CompileLog.FileSuccess(
+        outFileName(topClass.nameAsStr),
+        out.result
+      ))
+    )
+  }
+
+  def compileClass(cs: ClassSpec): Unit = {
+    out.puts(s"${type2class(cs)} = Struct(")
+    out.inc
+
+    cs.seq.foreach((seqAttr) => compileAttr(seqAttr))
+
+    out.dec
+    out.puts(")")
+  }
+
+  def compileAttr(attr: AttrSpec): Unit = {
+    out.puts(s"'${idToStr(attr.id)}' / ${typeToStr(attr.dataType)},")
+  }
+
+  def idToStr(id: Identifier): String = {
+    id match {
+      case SpecialIdentifier(name) => name
+      case NamedIdentifier(name) => name
+      case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx"
+      case InstanceIdentifier(name) => name
+    }
+  }
+
+  def type2class(cs: ClassSpec) = cs.name.last
+
+  def typeToStr(dataType: DataType): String = dataType match {
+    case Int1Type(signed) =>
+      s"Int8${if (signed) "s" else "u"}b"
+    case IntMultiType(signed, width, endianOpt) =>
+      s"Int${width.width * 8}${if (signed) "s" else "u"}${fixedEndianToStr(endianOpt.get)}"
+    case _ => "???"
+  }
+
+  def fixedEndianToStr(e: FixedEndian) = e match {
+    case LittleEndian => "l"
+    case BigEndian => "b"
+  }
+
+  def indent: String = "\t"
+  def outFileName(topClassName: String): String = s"$topClassName.py"
+}
+
+object ConstructClassCompiler extends LanguageCompilerStatic {
+  // FIXME: Unused, should be probably separated from LanguageCompilerStatic
+  override def getCompiler(tp: ClassTypeProvider, config: RuntimeConfig): LanguageCompiler = ???
+}
diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala
index 420775f38..d3f1374de 100644
--- a/shared/src/main/scala/io/kaitai/struct/Main.scala
+++ b/shared/src/main/scala/io/kaitai/struct/Main.scala
@@ -59,6 +59,8 @@ object Main {
         new GraphvizClassCompiler(specs, spec)
       case GoCompiler =>
         new GoClassCompiler(specs, spec, config)
+      case ConstructClassCompiler =>
+        new ConstructClassCompiler(specs, spec)
       case _ =>
         new ClassCompiler(specs, spec, config, lang)
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
index 93d0dc023..bc14394b9 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
@@ -10,6 +10,7 @@ trait LanguageCompilerStatic {
 
 object LanguageCompilerStatic {
   val NAME_TO_CLASS: Map[String, LanguageCompilerStatic] = Map(
+    "construct" -> ConstructClassCompiler,
     "cpp_stl" -> CppCompiler,
     "csharp" -> CSharpCompiler,
     "graphviz" -> GraphvizClassCompiler,

From 14c231af597ae1f7695f4397539b9629c3e746fe Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Mar 2018 20:07:29 +0000
Subject: [PATCH 072/230] ConstructClassCompiler: added some basic strings
 support

---
 .../io/kaitai/struct/ConstructClassCompiler.scala   | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index 869ab5642..615b2370e 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -62,6 +62,19 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
       s"Int8${if (signed) "s" else "u"}b"
     case IntMultiType(signed, width, endianOpt) =>
       s"Int${width.width * 8}${if (signed) "s" else "u"}${fixedEndianToStr(endianOpt.get)}"
+    case StrFromBytesType(bytes, encoding) =>
+      bytes match {
+        case BytesEosType(terminator, include, padRight, process) =>
+          s"GreedyString(encoding='$encoding')"
+        case BytesLimitType(size, terminator, include, padRight, process) =>
+          s"FixedSized(${translator.translate(size)}, GreedyString(encoding='$encoding'))"
+        case BytesTerminatedType(terminator, include, consume, eosError, process) =>
+          val termStr = "\\x%02X".format(terminator & 0xff)
+          s"NullTerminated(GreedyString(encoding='$encoding'), " +
+            s"term=b'$termStr', " +
+            s"include=${translator.doBoolLiteral(include)}, " +
+            s"consume=${translator.doBoolLiteral(consume)})"
+      }
     case _ => "???"
   }
 

From da5ea4ebcad152311df8e2f5f3f4755733e56382 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 22 Mar 2018 22:25:26 +0000
Subject: [PATCH 073/230] ConstructClassCompiler: enums and user types should
 work

---
 .../struct/ConstructClassCompiler.scala       | 36 +++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index 615b2370e..adb41e7b9 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -20,7 +20,6 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
 
     compileClass(topClass)
 
-    out.puts
     out.puts(s"_schema = ${type2class(topClass)}")
 
     CompileLog.SpecSuccess(
@@ -33,6 +32,10 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
   }
 
   def compileClass(cs: ClassSpec): Unit = {
+    cs.types.foreach { case (_, typeSpec) => compileClass(typeSpec) }
+
+    cs.enums.foreach { case (_, enumSpec) => compileEnum(enumSpec) }
+
     out.puts(s"${type2class(cs)} = Struct(")
     out.inc
 
@@ -40,12 +43,27 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
 
     out.dec
     out.puts(")")
+    out.puts
   }
 
   def compileAttr(attr: AttrSpec): Unit = {
     out.puts(s"'${idToStr(attr.id)}' / ${typeToStr(attr.dataType)},")
   }
 
+  def compileEnum(enumSpec: EnumSpec): Unit = {
+    out.puts(s"def ${enumToStr(enumSpec)}(subcon):")
+    out.inc
+    out.puts("return Enum(subcon,")
+    out.inc
+    enumSpec.sortedSeq.foreach { case (number, valueSpec) =>
+      out.puts(s"${valueSpec.name}=$number,")
+    }
+    out.dec
+    out.puts(")")
+    out.dec
+    out.puts
+  }
+
   def idToStr(id: Identifier): String = {
     id match {
       case SpecialIdentifier(name) => name
@@ -55,7 +73,9 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
     }
   }
 
-  def type2class(cs: ClassSpec) = cs.name.last
+  def type2class(cs: ClassSpec) = cs.name.mkString("__")
+
+  def enumToStr(enumSpec: EnumSpec) = enumSpec.name.mkString("__")
 
   def typeToStr(dataType: DataType): String = dataType match {
     case Int1Type(signed) =>
@@ -75,6 +95,18 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
             s"include=${translator.doBoolLiteral(include)}, " +
             s"consume=${translator.doBoolLiteral(consume)})"
       }
+    case ut: UserTypeInstream =>
+      type2class(ut.classSpec.get)
+    case utb: UserTypeFromBytes =>
+      utb.bytes match {
+        //case BytesEosType(terminator, include, padRight, process) =>
+        case BytesLimitType(size, terminator, include, padRight, process) =>
+          s"FixedSized(${translator.translate(size)}, ${type2class(utb.classSpec.get)})"
+        //case BytesTerminatedType(terminator, include, consume, eosError, process) =>
+        case _ => "???"
+      }
+    case et: EnumType =>
+      s"${enumToStr(et.enumSpec.get)}(${typeToStr(et.basedOn)})"
     case _ => "???"
   }
 

From a381c445efe278ebb17b3c886544b992cf99263a Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 23 Mar 2018 09:57:30 +0000
Subject: [PATCH 074/230] Construct: basic support for instances, floats,
 extracted signToStr

---
 .../struct/ConstructClassCompiler.scala       | 24 ++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index adb41e7b9..2ad08d6e7 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -40,16 +40,28 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
     out.inc
 
     cs.seq.foreach((seqAttr) => compileAttr(seqAttr))
+    cs.instances.foreach { case (id, instSpec) =>
+      instSpec match {
+        case vis: ValueInstanceSpec =>
+          compileValueInstance(id, vis)
+        case pis: ParseInstanceSpec =>
+          compileAttr(pis)
+      }
+    }
 
     out.dec
     out.puts(")")
     out.puts
   }
 
-  def compileAttr(attr: AttrSpec): Unit = {
+  def compileAttr(attr: AttrLikeSpec): Unit = {
     out.puts(s"'${idToStr(attr.id)}' / ${typeToStr(attr.dataType)},")
   }
 
+  def compileValueInstance(id: Identifier, vis: ValueInstanceSpec): Unit = {
+    out.puts(s"'${idToStr(id)}' / Computed(${translator.translate(vis.value)}),")
+  }
+
   def compileEnum(enumSpec: EnumSpec): Unit = {
     out.puts(s"def ${enumToStr(enumSpec)}(subcon):")
     out.inc
@@ -78,10 +90,14 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
   def enumToStr(enumSpec: EnumSpec) = enumSpec.name.mkString("__")
 
   def typeToStr(dataType: DataType): String = dataType match {
+    case fbt: FixedBytesType =>
+      s"Const(${translator.doByteArrayLiteral(fbt.contents)})"
     case Int1Type(signed) =>
-      s"Int8${if (signed) "s" else "u"}b"
+      s"Int8${signToStr(signed)}b"
     case IntMultiType(signed, width, endianOpt) =>
-      s"Int${width.width * 8}${if (signed) "s" else "u"}${fixedEndianToStr(endianOpt.get)}"
+      s"Int${width.width * 8}${signToStr(signed)}${fixedEndianToStr(endianOpt.get)}"
+    case FloatMultiType(width, endianOpt) =>
+      s"Float${width.width * 8}${fixedEndianToStr(endianOpt.get)}"
     case StrFromBytesType(bytes, encoding) =>
       bytes match {
         case BytesEosType(terminator, include, padRight, process) =>
@@ -110,6 +126,8 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
     case _ => "???"
   }
 
+  def signToStr(signed: Boolean) = if (signed) "s" else "u"
+
   def fixedEndianToStr(e: FixedEndian) = e match {
     case LittleEndian => "l"
     case BigEndian => "b"

From 039e200134613cc53cfdde65de44c9c933341ee2 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 23 Mar 2018 10:49:39 +0000
Subject: [PATCH 075/230] Construct: basic array support

---
 .../struct/ConstructClassCompiler.scala       | 35 +++++++++++++++----
 1 file changed, 28 insertions(+), 7 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index 2ad08d6e7..bade04bb9 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -55,7 +55,18 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
   }
 
   def compileAttr(attr: AttrLikeSpec): Unit = {
-    out.puts(s"'${idToStr(attr.id)}' / ${typeToStr(attr.dataType)},")
+    val typeStr1 = typeToStr(attr.dataType)
+    val typeStr2 = attr.cond.repeat match {
+      case RepeatExpr(expr) =>
+        s"Array(${translator.translate(expr)}, $typeStr1)"
+      case RepeatUntil(expr) =>
+        "???"
+      case RepeatEos =>
+        s"GreedyRange($typeStr1)"
+      case NoRepeat =>
+        typeStr1
+    }
+    out.puts(s"'${idToStr(attr.id)}' / $typeStr2,")
   }
 
   def compileValueInstance(id: Identifier, vis: ValueInstanceSpec): Unit = {
@@ -98,18 +109,20 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
       s"Int${width.width * 8}${signToStr(signed)}${fixedEndianToStr(endianOpt.get)}"
     case FloatMultiType(width, endianOpt) =>
       s"Float${width.width * 8}${fixedEndianToStr(endianOpt.get)}"
+    case BytesEosType(terminator, include, padRight, process) =>
+      "GreedyBytes"
+    case BytesLimitType(size, terminator, include, padRight, process) =>
+      s"Bytes(${translator.translate(size)})"
+    case btt: BytesTerminatedType =>
+      attrBytesTerminatedType(btt, "GreedyBytes")
     case StrFromBytesType(bytes, encoding) =>
       bytes match {
         case BytesEosType(terminator, include, padRight, process) =>
           s"GreedyString(encoding='$encoding')"
         case BytesLimitType(size, terminator, include, padRight, process) =>
           s"FixedSized(${translator.translate(size)}, GreedyString(encoding='$encoding'))"
-        case BytesTerminatedType(terminator, include, consume, eosError, process) =>
-          val termStr = "\\x%02X".format(terminator & 0xff)
-          s"NullTerminated(GreedyString(encoding='$encoding'), " +
-            s"term=b'$termStr', " +
-            s"include=${translator.doBoolLiteral(include)}, " +
-            s"consume=${translator.doBoolLiteral(consume)})"
+        case btt: BytesTerminatedType =>
+          attrBytesTerminatedType(btt, s"GreedyString(encoding='$encoding')")
       }
     case ut: UserTypeInstream =>
       type2class(ut.classSpec.get)
@@ -126,6 +139,14 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
     case _ => "???"
   }
 
+  def attrBytesTerminatedType(btt: BytesTerminatedType, subcon: String): String = {
+    val termStr = "\\x%02X".format(btt.terminator & 0xff)
+    s"NullTerminated($subcon, " +
+      s"term=b'$termStr', " +
+      s"include=${translator.doBoolLiteral(btt.include)}, " +
+      s"consume=${translator.doBoolLiteral(btt.consume)})"
+  }
+
   def signToStr(signed: Boolean) = if (signed) "s" else "u"
 
   def fixedEndianToStr(e: FixedEndian) = e match {

From 2661f083decc2af3cbc9f692f9631028cadefe05 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 23 Mar 2018 19:51:38 +0000
Subject: [PATCH 076/230] Construct: replaced PythonTranslator with
 ConstructTranslator; the only difference now is that it generates `this.`
 instead of `self.`

---
 .../io/kaitai/struct/ConstructClassCompiler.scala  |  6 +++---
 .../struct/translators/ConstructTranslator.scala   | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)
 create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index bade04bb9..c721add73 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -1,17 +1,17 @@
 package io.kaitai.struct
 
-import io.kaitai.struct.datatype._
 import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.datatype._
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic}
-import io.kaitai.struct.translators.PythonTranslator
+import io.kaitai.struct.translators.ConstructTranslator
 
 class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends AbstractCompiler {
   val out = new StringLanguageOutputWriter(indent)
   val importList = new ImportList
 
   val provider = new ClassTypeProvider(classSpecs, topClass)
-  val translator = new PythonTranslator(provider, importList)
+  val translator = new ConstructTranslator(provider, importList)
 
   override def compile: CompileLog.SpecSuccess = {
     out.puts("from construct import *")
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
new file mode 100644
index 000000000..f4e7cdc18
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
@@ -0,0 +1,14 @@
+package io.kaitai.struct.translators
+
+import io.kaitai.struct.ImportList
+import io.kaitai.struct.format.Identifier
+
+class ConstructTranslator(provider: TypeProvider, importList: ImportList) extends PythonTranslator(provider, importList) {
+  override def doLocalName(s: String) = {
+    s match {
+      case Identifier.ITERATOR => "_"
+      case Identifier.INDEX => "i"
+      case _ => s"this.${doName(s)}"
+    }
+  }
+}

From 214cdf44dfb9967846af62c9c5ea6fcb26b26cf8 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 24 Mar 2018 12:14:10 +0000
Subject: [PATCH 077/230] Construct: added `if` support, proper `pos` support,
 LazyBound type resolution, proper nowClass setting

---
 .../struct/ConstructClassCompiler.scala       | 58 ++++++++++++++-----
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index c721add73..dd9abbfb5 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -2,6 +2,7 @@ package io.kaitai.struct
 
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.datatype._
+import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic}
 import io.kaitai.struct.translators.ConstructTranslator
@@ -39,13 +40,15 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
     out.puts(s"${type2class(cs)} = Struct(")
     out.inc
 
+    provider.nowClass = cs
+
     cs.seq.foreach((seqAttr) => compileAttr(seqAttr))
     cs.instances.foreach { case (id, instSpec) =>
       instSpec match {
         case vis: ValueInstanceSpec =>
           compileValueInstance(id, vis)
         case pis: ParseInstanceSpec =>
-          compileAttr(pis)
+          compileParseInstance(pis)
       }
     }
 
@@ -55,22 +58,45 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
   }
 
   def compileAttr(attr: AttrLikeSpec): Unit = {
-    val typeStr1 = typeToStr(attr.dataType)
-    val typeStr2 = attr.cond.repeat match {
-      case RepeatExpr(expr) =>
-        s"Array(${translator.translate(expr)}, $typeStr1)"
-      case RepeatUntil(expr) =>
-        "???"
-      case RepeatEos =>
-        s"GreedyRange($typeStr1)"
-      case NoRepeat =>
-        typeStr1
-    }
-    out.puts(s"'${idToStr(attr.id)}' / $typeStr2,")
+    out.puts(s"'${idToStr(attr.id)}' / ${compileAttrBody(attr)},")
   }
 
   def compileValueInstance(id: Identifier, vis: ValueInstanceSpec): Unit = {
-    out.puts(s"'${idToStr(id)}' / Computed(${translator.translate(vis.value)}),")
+    val typeStr = s"Computed(${translator.translate(vis.value)})"
+    val typeStr2 = wrapWithIf(typeStr, vis.ifExpr)
+    out.puts(s"'${idToStr(id)}' / $typeStr2,")
+  }
+
+  def compileParseInstance(attr: ParseInstanceSpec): Unit = {
+    attr.pos match {
+      case None =>
+        compileAttr(attr)
+      case Some(pos) =>
+        out.puts(s"'${idToStr(attr.id)}' / " +
+          s"Pointer(${translator.translate(pos)}, ${compileAttrBody(attr)}),")
+    }
+  }
+
+  def compileAttrBody(attr: AttrLikeSpec): String = {
+    val typeStr1 = typeToStr(attr.dataType)
+    val typeStr2 = wrapWithRepeat(typeStr1, attr.cond.repeat)
+    wrapWithIf(typeStr2, attr.cond.ifExpr)
+  }
+
+  def wrapWithRepeat(typeStr: String, repeat: RepeatSpec) = repeat match {
+    case RepeatExpr(expr) =>
+      s"Array(${translator.translate(expr)}, $typeStr)"
+    case RepeatUntil(expr) =>
+      "???"
+    case RepeatEos =>
+      s"GreedyRange($typeStr)"
+    case NoRepeat =>
+      typeStr
+  }
+
+  def wrapWithIf(typeStr: String, ifExpr: Option[Ast.expr]) = ifExpr match {
+    case Some(expr) => s"If(${translator.translate(expr)}, $typeStr)"
+    case None => typeStr
   }
 
   def compileEnum(enumSpec: EnumSpec): Unit = {
@@ -125,12 +151,12 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
           attrBytesTerminatedType(btt, s"GreedyString(encoding='$encoding')")
       }
     case ut: UserTypeInstream =>
-      type2class(ut.classSpec.get)
+      s"LazyBound(lambda: ${type2class(ut.classSpec.get)})"
     case utb: UserTypeFromBytes =>
       utb.bytes match {
         //case BytesEosType(terminator, include, padRight, process) =>
         case BytesLimitType(size, terminator, include, padRight, process) =>
-          s"FixedSized(${translator.translate(size)}, ${type2class(utb.classSpec.get)})"
+          s"FixedSized(${translator.translate(size)}, LazyBound(lambda: ${type2class(utb.classSpec.get)}))"
         //case BytesTerminatedType(terminator, include, consume, eosError, process) =>
         case _ => "???"
       }

From cfeda532ac6db6b1d08f7cddeb28be55aabf54db Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 26 Mar 2018 01:17:43 +0100
Subject: [PATCH 078/230] Construct: fix _root, _parent and _io handling

---
 .../kaitai/struct/translators/ConstructTranslator.scala  | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
index f4e7cdc18..ca4d492c7 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
@@ -8,7 +8,16 @@ class ConstructTranslator(provider: TypeProvider, importList: ImportList) extend
     s match {
       case Identifier.ITERATOR => "_"
       case Identifier.INDEX => "i"
+      case Identifier.ROOT => "_root"
+      case Identifier.IO => "_stream"
       case _ => s"this.${doName(s)}"
     }
   }
+
+  override def doName(s: String) = {
+    s match {
+      case Identifier.PARENT => "_"
+      case _ => s
+    }
+  }
 }

From bfc33ca1735eb64309e0293d671b665e4c717e5c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 26 Mar 2018 01:37:31 +0100
Subject: [PATCH 079/230] Construct: fix terminator / pad-right translation

---
 .../struct/ConstructClassCompiler.scala       | 24 +++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index dd9abbfb5..ddd4d1387 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -137,16 +137,16 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
       s"Float${width.width * 8}${fixedEndianToStr(endianOpt.get)}"
     case BytesEosType(terminator, include, padRight, process) =>
       "GreedyBytes"
-    case BytesLimitType(size, terminator, include, padRight, process) =>
-      s"Bytes(${translator.translate(size)})"
+    case blt: BytesLimitType =>
+      attrBytesLimitType(blt)
     case btt: BytesTerminatedType =>
       attrBytesTerminatedType(btt, "GreedyBytes")
     case StrFromBytesType(bytes, encoding) =>
       bytes match {
         case BytesEosType(terminator, include, padRight, process) =>
           s"GreedyString(encoding='$encoding')"
-        case BytesLimitType(size, terminator, include, padRight, process) =>
-          s"FixedSized(${translator.translate(size)}, GreedyString(encoding='$encoding'))"
+        case blt: BytesLimitType =>
+          attrBytesLimitType(blt, s"GreedyString(encoding='$encoding')")
         case btt: BytesTerminatedType =>
           attrBytesTerminatedType(btt, s"GreedyString(encoding='$encoding')")
       }
@@ -165,6 +165,22 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
     case _ => "???"
   }
 
+  def attrBytesLimitType(blt: BytesLimitType, subcon: String = "GreedyBytes"): String = {
+    val subcon2 = blt.terminator match {
+      case None => subcon
+      case Some(term) =>
+        val termStr = "\\x%02X".format(term & 0xff)
+        s"NullTerminated($subcon, term=b'$termStr', include=${translator.doBoolLiteral(blt.include)})"
+    }
+    val subcon3 = blt.padRight match {
+      case None => subcon2
+      case Some(padRight) =>
+        val padStr = "\\x%02X".format(padRight & 0xff)
+        s"NullStripped($subcon2, pad=b'$padStr')"
+    }
+    s"FixedSized(${translator.translate(blt.size)}, $subcon3)"
+  }
+
   def attrBytesTerminatedType(btt: BytesTerminatedType, subcon: String): String = {
     val termStr = "\\x%02X".format(btt.terminator & 0xff)
     s"NullTerminated($subcon, " +

From eb82243a7fd2b217b2531a01c6a933304f15b3ed Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 26 Mar 2018 18:39:36 +0100
Subject: [PATCH 080/230] Construct: implement repeat-until

---
 .../scala/io/kaitai/struct/ConstructClassCompiler.scala    | 7 ++++---
 .../io/kaitai/struct/translators/ConstructTranslator.scala | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index ddd4d1387..5075f15b2 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -79,15 +79,16 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
 
   def compileAttrBody(attr: AttrLikeSpec): String = {
     val typeStr1 = typeToStr(attr.dataType)
-    val typeStr2 = wrapWithRepeat(typeStr1, attr.cond.repeat)
+    val typeStr2 = wrapWithRepeat(typeStr1, attr.cond.repeat, attr.dataType)
     wrapWithIf(typeStr2, attr.cond.ifExpr)
   }
 
-  def wrapWithRepeat(typeStr: String, repeat: RepeatSpec) = repeat match {
+  def wrapWithRepeat(typeStr: String, repeat: RepeatSpec, dataType: DataType) = repeat match {
     case RepeatExpr(expr) =>
       s"Array(${translator.translate(expr)}, $typeStr)"
     case RepeatUntil(expr) =>
-      "???"
+      provider._currentIteratorType = Some(dataType)
+      s"RepeatUntil(lambda obj_, list_, this: ${translator.translate(expr)}, $typeStr)"
     case RepeatEos =>
       s"GreedyRange($typeStr)"
     case NoRepeat =>
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
index ca4d492c7..9c11839a5 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
@@ -6,7 +6,7 @@ import io.kaitai.struct.format.Identifier
 class ConstructTranslator(provider: TypeProvider, importList: ImportList) extends PythonTranslator(provider, importList) {
   override def doLocalName(s: String) = {
     s match {
-      case Identifier.ITERATOR => "_"
+      case Identifier.ITERATOR => "obj_"
       case Identifier.INDEX => "i"
       case Identifier.ROOT => "_root"
       case Identifier.IO => "_stream"

From d80065419deed23542d2f8d6290bc656e29daa1a Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 27 Mar 2018 10:38:31 +0100
Subject: [PATCH 081/230] Construct: Computed with mandatory lambda and
 implemented SwitchType output

---
 .../io/kaitai/struct/ConstructClassCompiler.scala     | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index 5075f15b2..0b55d5340 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -62,7 +62,7 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
   }
 
   def compileValueInstance(id: Identifier, vis: ValueInstanceSpec): Unit = {
-    val typeStr = s"Computed(${translator.translate(vis.value)})"
+    val typeStr = s"Computed(lambda this: ${translator.translate(vis.value)})"
     val typeStr2 = wrapWithIf(typeStr, vis.ifExpr)
     out.puts(s"'${idToStr(id)}' / $typeStr2,")
   }
@@ -163,6 +163,8 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
       }
     case et: EnumType =>
       s"${enumToStr(et.enumSpec.get)}(${typeToStr(et.basedOn)})"
+    case st: SwitchType =>
+      attrSwitchType(st)
     case _ => "???"
   }
 
@@ -190,6 +192,13 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
       s"consume=${translator.doBoolLiteral(btt.consume)})"
   }
 
+  def attrSwitchType(st: SwitchType): String = {
+    s"Switch(${translator.translate(st.on)}, {" +
+      st.cases.map { case (caseExpr, caseType) =>
+          s"${translator.translate(caseExpr)}: ${typeToStr(caseType)}, "
+      }.mkString + "})"
+  }
+
   def signToStr(signed: Boolean) = if (signed) "s" else "u"
 
   def fixedEndianToStr(e: FixedEndian) = e match {

From dcf2fd2dca5f3a2829eebfb4176cc13c50ecce40 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 28 Mar 2018 02:15:50 +0100
Subject: [PATCH 082/230] Construct: implemented default for switching types

---
 .../io/kaitai/struct/ConstructClassCompiler.scala | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
index 0b55d5340..a4b6a3c53 100644
--- a/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ConstructClassCompiler.scala
@@ -193,10 +193,17 @@ class ConstructClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extend
   }
 
   def attrSwitchType(st: SwitchType): String = {
-    s"Switch(${translator.translate(st.on)}, {" +
-      st.cases.map { case (caseExpr, caseType) =>
-          s"${translator.translate(caseExpr)}: ${typeToStr(caseType)}, "
-      }.mkString + "})"
+    val cases = st.cases.filter { case (caseExpr, _) =>
+      caseExpr != SwitchType.ELSE_CONST
+    }.map { case (caseExpr, caseType) =>
+      s"${translator.translate(caseExpr)}: ${typeToStr(caseType)}, "
+    }
+
+    val defaultSuffix = st.cases.get(SwitchType.ELSE_CONST).map((t) =>
+      s", default=${typeToStr(t)}"
+    ).getOrElse("")
+
+    s"Switch(${translator.translate(st.on)}, {${cases.mkString}}$defaultSuffix)"
   }
 
   def signToStr(signed: Boolean) = if (signed) "s" else "u"

From ee4080e4bc7fbb85bb421979cf6974c6a2868c1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Thu, 29 Mar 2018 21:20:53 +0200
Subject: [PATCH 083/230] Fix wrong iterator handling for user types in
 repeat-until.

---
 .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala    | 2 +-
 .../main/scala/io/kaitai/struct/languages/RubyCompiler.scala    | 1 +
 .../scala/io/kaitai/struct/translators/RubyTranslator.scala     | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index c20b1781d..2aa7ce98e 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -389,7 +389,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"${privateMemberName(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}();")
     out.puts("{")
     out.inc
-    out.puts(s"${kaitaiType2JavaType(dataType)} ${translator.doName("_")};")
+    out.puts(s"${kaitaiType2JavaType(dataType)} ${translator.doName(Identifier.ITERATOR)} = null;")
     out.puts("int i = 0;")
     out.puts("do {")
     out.inc
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index 9138981e1..8ef63265b 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -308,6 +308,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     if (needRaw)
       out.puts(s"${privateMemberName(RawIdentifier(id))} = []")
     out.puts(s"${privateMemberName(id)} = []")
+    out.puts(s"${translator.doName(Identifier.ITERATOR)} = nil")
     out.puts("i = 0")
     out.puts("begin")
     out.inc
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
index 52975cee5..21fa3dd13 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -29,6 +29,7 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
 
   override def doName(s: String) = {
     s match {
+      case Identifier.ITERATOR => "_it"
       case Identifier.INDEX => "i" // FIXME: probably would clash with attribute named "i"
       case _ => s
     }

From 54c74d05f5bb466ffe6c22544c1b644b48c45fae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <tschoening@users.noreply.github.com>
Date: Fri, 30 Mar 2018 09:49:49 +0200
Subject: [PATCH 084/230] Remove warning about unused i in Java. This fixes
 https://github.com/kaitai-io/kaitai_struct/issues/365 in a somewhat easy
 manner and can be improved in future if needed at all.

---
 .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 2aa7ce98e..c3274aa6b 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -362,7 +362,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def condRepeatEosFooter: Unit = {
-    out.puts("i++;")
+    out.puts("i = i + 1;")
     out.dec
     out.puts("}")
     out.dec
@@ -409,7 +409,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = {
     typeProvider._currentIteratorType = Some(dataType)
-    out.puts("i++;")
+    out.puts("i = i + 1;")
     out.dec
     out.puts(s"} while (!(${expression(untilExpr)}));")
     out.dec

From 7f50fb5713cd5f4aa7f4abc9c789f3b91ed10282 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 31 Mar 2018 10:43:51 +0100
Subject: [PATCH 085/230] Construct: fixed stream methods invocations as
 recommended by @arekbulski

---
 .../struct/translators/ConstructTranslator.scala       | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
index 9c11839a5..689c71aeb 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
@@ -1,6 +1,7 @@
 package io.kaitai.struct.translators
 
 import io.kaitai.struct.ImportList
+import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.Identifier
 
 class ConstructTranslator(provider: TypeProvider, importList: ImportList) extends PythonTranslator(provider, importList) {
@@ -9,7 +10,7 @@ class ConstructTranslator(provider: TypeProvider, importList: ImportList) extend
       case Identifier.ITERATOR => "obj_"
       case Identifier.INDEX => "i"
       case Identifier.ROOT => "_root"
-      case Identifier.IO => "_stream"
+      case Identifier.IO => "_io"
       case _ => s"this.${doName(s)}"
     }
   }
@@ -20,4 +21,11 @@ class ConstructTranslator(provider: TypeProvider, importList: ImportList) extend
       case _ => s
     }
   }
+
+  override def kaitaiStreamSize(value: Ast.expr): String =
+    s"stream_size(${translate(value)})"
+  override def kaitaiStreamEof(value: Ast.expr): String =
+    s"stream_iseof(${translate(value)})"
+  override def kaitaiStreamPos(value: Ast.expr): String =
+    s"stream_tell(${translate(value)})"
 }

From 531ca8f75bacdceb876198fc1fe039b10278ab08 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 31 Mar 2018 10:58:07 +0100
Subject: [PATCH 086/230] TypeDetector: extract detectAttributeType &
 detectCallType into distinct methods

---
 .../struct/translators/TypeDetector.scala     | 153 ++++++++++--------
 1 file changed, 88 insertions(+), 65 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index ba19f4479..658f9b7e4 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -3,7 +3,6 @@ package io.kaitai.struct.translators
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
-import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.precompile.{TypeMismatchError, TypeUndecidedError}
 
 /**
@@ -90,7 +89,7 @@ class TypeDetector(provider: TypeProvider) {
           }
         })
         CalcBooleanType
-      case Ast.expr.IfExp(condition: expr, ifTrue: expr, ifFalse: expr) =>
+      case Ast.expr.IfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr) =>
         detectType(condition) match {
           case _: BooleanType =>
             val trueType = detectType(ifTrue)
@@ -108,68 +107,9 @@ class TypeDetector(provider: TypeProvider) {
           case cntType => throw new TypeMismatchError(s"unable to apply operation [] to $cntType")
         }
       case Ast.expr.Attribute(value: Ast.expr, attr: Ast.identifier) =>
-        val valType = detectType(value)
-        valType match {
-          case KaitaiStructType =>
-            throw new TypeMismatchError(s"called attribute '${attr.name}' on generic struct expression '$value'")
-          case t: UserType =>
-            t.classSpec match {
-              case Some(tt) => provider.determineType(tt, attr.name)
-              case None => throw new TypeUndecidedError(s"expression '$value' has undecided type '${t.name}' (while asking for attribute '${attr.name}')")
-            }
-          case _: StrType =>
-            attr.name match {
-              case "length" => CalcIntType
-              case "reverse" => CalcStrType
-              case "to_i" => CalcIntType
-              case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
-            }
-          case _: IntType =>
-            attr.name match {
-              case "to_s" => CalcStrType
-              case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
-            }
-          case _: FloatType =>
-            attr.name match {
-              case "to_i" => CalcIntType
-              case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
-            }
-          case ArrayType(inType) =>
-            attr.name match {
-              case "first" | "last" | "min" | "max" => inType
-              case "size" => CalcIntType
-              case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
-            }
-          case KaitaiStreamType =>
-            attr.name match {
-              case "size" => CalcIntType
-              case "pos" => CalcIntType
-              case "eof" => CalcBooleanType
-              case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
-            }
-          case et: EnumType =>
-            attr.name match {
-              case "to_i" => CalcIntType
-              case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
-            }
-          case _: BooleanType =>
-            attr.name match {
-              case "to_i" => CalcIntType
-              case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
-            }
-          case _ =>
-            throw new TypeMismatchError(s"don't know how to call anything on $valType")
-        }
-      case Ast.expr.Call(func: Ast.expr, args: Seq[Ast.expr]) =>
-        func match {
-          case Ast.expr.Attribute(obj: Ast.expr, methodName: Ast.identifier) =>
-            val objType = detectType(obj)
-            (objType, methodName.name) match {
-              case (_: StrType, "substring") => CalcStrType
-              case (_: StrType, "to_i") => CalcIntType
-              case _ => throw new RuntimeException(s"don't know how to call method '$methodName' of object type '$objType'")
-            }
-        }
+        detectAttributeType(value, attr)
+      case call: Ast.expr.Call =>
+        detectCallType(call)
       case Ast.expr.List(values: Seq[Ast.expr]) =>
         detectArrayType(values) match {
           case Int1Type(_) => CalcBytesType
@@ -180,6 +120,89 @@ class TypeDetector(provider: TypeProvider) {
     }
   }
 
+  /**
+    * Detects resulting data type of a given attribute expression.
+    * @note Must be kept in sync with [[CommonMethods.translateAttribute]]
+    * @param value value part of attribute expression
+    * @param attr attribute identifier part of attribute expression
+    * @return data type
+    */
+  def detectAttributeType(value: Ast.expr, attr: Ast.identifier) = {
+    val valType = detectType(value)
+    valType match {
+      case KaitaiStructType =>
+        throw new TypeMismatchError(s"called attribute '${attr.name}' on generic struct expression '$value'")
+      case t: UserType =>
+        t.classSpec match {
+          case Some(tt) => provider.determineType(tt, attr.name)
+          case None => throw new TypeUndecidedError(s"expression '$value' has undecided type '${t.name}' (while asking for attribute '${attr.name}')")
+        }
+      case _: StrType =>
+        attr.name match {
+          case "length" => CalcIntType
+          case "reverse" => CalcStrType
+          case "to_i" => CalcIntType
+          case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+        }
+      case _: IntType =>
+        attr.name match {
+          case "to_s" => CalcStrType
+          case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+        }
+      case _: FloatType =>
+        attr.name match {
+          case "to_i" => CalcIntType
+          case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+        }
+      case ArrayType(inType) =>
+        attr.name match {
+          case "first" | "last" | "min" | "max" => inType
+          case "size" => CalcIntType
+          case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+        }
+      case KaitaiStreamType =>
+        attr.name match {
+          case "size" => CalcIntType
+          case "pos" => CalcIntType
+          case "eof" => CalcBooleanType
+          case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+        }
+      case et: EnumType =>
+        attr.name match {
+          case "to_i" => CalcIntType
+          case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+        }
+      case _: BooleanType =>
+        attr.name match {
+          case "to_i" => CalcIntType
+          case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+        }
+      case _ =>
+        throw new TypeMismatchError(s"don't know how to call anything on $valType")
+    }
+  }
+
+  /**
+    * Detects resulting data type of a given function call expression. Typical function
+    * call expression in KSY is `foo.bar(arg1, arg2)`, which is represented in AST as
+    * `Call(Attribute(foo, bar), Seq(arg1, arg2))`.
+    * @note Must be kept in sync with [[CommonMethods.translateCall]]
+    * @param call function call expression
+    * @return data type
+    */
+  def detectCallType(call: Ast.expr.Call): DataType = {
+    call.func match {
+      case Ast.expr.Attribute(obj: Ast.expr, methodName: Ast.identifier) =>
+        val objType = detectType(obj)
+        // TODO: check number and type of arguments in `call.args`
+        (objType, methodName.name) match {
+          case (_: StrType, "substring") => CalcStrType
+          case (_: StrType, "to_i") => CalcIntType
+          case _ => throw new RuntimeException(s"don't know how to call method '$methodName' of object type '$objType'")
+        }
+    }
+  }
+
   /**
     * Checks that elements in the array literal are all the of same type and
     * returns that type. Throws exception if multiple mismatching types are
@@ -188,7 +211,7 @@ class TypeDetector(provider: TypeProvider) {
     * @param values
     * @return
     */
-  def detectArrayType(values: Seq[expr]): DataType = {
+  def detectArrayType(values: Seq[Ast.expr]): DataType = {
     var t1o: Option[DataType] = None
 
     values.foreach { v =>

From 9e8175f282e98b4a6dcf07daa0c2f86ee44a385c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 31 Mar 2018 11:00:37 +0100
Subject: [PATCH 087/230] Added `.length` for byte arrays, and `bytesLength`
 common method that would needs to be eventually implemented everywhere

---
 .../struct/translators/CommonMethods.scala    | 20 +++++++++++++++++++
 .../struct/translators/TypeDetector.scala     |  5 +++++
 2 files changed, 25 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala
index b71faa868..511080510 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonMethods.scala
@@ -5,6 +5,13 @@ import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.precompile.TypeMismatchError
 
 abstract trait CommonMethods[T] extends TypeDetector {
+  /**
+    * Translates a certain attribute call (as in `foo.bar`) into a rendition
+    * of expression in certain target language.
+    * @note Must be kept in sync with [[TypeDetector.detectAttributeType]]
+    * @param call attribute call expression to translate
+    * @return result of translation as [[T]]
+    */
   def translateAttribute(call: Ast.expr.Attribute): T = {
     val attr = call.attr
     val value = call.value
@@ -12,6 +19,10 @@ abstract trait CommonMethods[T] extends TypeDetector {
     valType match {
       case ut: UserType =>
         userTypeField(ut, value, attr.name)
+      case _: BytesType =>
+        attr.name match {
+          case "length" => bytesLength(value)
+        }
       case _: StrType =>
         attr.name match {
           case "length" => strLength(value)
@@ -53,6 +64,13 @@ abstract trait CommonMethods[T] extends TypeDetector {
     }
   }
 
+  /**
+    * Translates a certain function call (as in `foo.bar(arg1, arg2)`) into a
+    * rendition of expression in certain target language.
+    * @note Must be kept in sync with [[TypeDetector.detectCallType]]
+    * @param call function call expression to translate
+    * @return result of translation as [[T]]
+    */
   def translateCall(call: Ast.expr.Call): T = {
     val func = call.func
     val args = call.args
@@ -72,6 +90,8 @@ abstract trait CommonMethods[T] extends TypeDetector {
 
   def userTypeField(ut: UserType, value: Ast.expr, name: String): T
 
+  def bytesLength(b: Ast.expr): T = ???
+
   def strLength(s: Ast.expr): T
   def strReverse(s: Ast.expr): T
   def strToInt(s: Ast.expr, base: Ast.expr): T
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index 658f9b7e4..9637aa696 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -137,6 +137,11 @@ class TypeDetector(provider: TypeProvider) {
           case Some(tt) => provider.determineType(tt, attr.name)
           case None => throw new TypeUndecidedError(s"expression '$value' has undecided type '${t.name}' (while asking for attribute '${attr.name}')")
         }
+      case _: BytesType =>
+        attr.name match {
+          case "length" => CalcIntType
+          case _ => throw new TypeMismatchError(s"called invalid attribute '${attr.name}' on expression of type $valType")
+        }
       case _: StrType =>
         attr.name match {
           case "length" => CalcIntType

From c844fd04109423c2a9ab187221c4461b92a45b17 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 31 Mar 2018 13:14:13 +0100
Subject: [PATCH 088/230] Updated 0.8 release notes, started 0.9 release notes

---
 RELEASE_NOTES.md | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 3943451bb..5b044839c 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,4 +1,12 @@
-# 0.8 (TBD)
+# 0.9 (TBD)
+
+* New targets support:
+  * Python with [Construct library](https://construct.readthedocs.io)
+* Expression language:
+  * New methods:
+    * byte arrays: `length`
+
+# 0.8 (2018-02-05)
 
 * New target languages:
   * Lua (96% tests pass score)

From f90425039236b71c344f3626787fe2e46da72d96 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 31 Mar 2018 13:45:15 +0100
Subject: [PATCH 089/230] Added bytes length test, added test tags

---
 .../kaitai/struct/translators/TranslatorSpec.scala | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 1f301bbb3..312f31857 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -7,7 +7,7 @@ import io.kaitai.struct.format.ClassSpec
 import io.kaitai.struct.languages._
 import io.kaitai.struct.languages.components.LanguageCompilerStatic
 import io.kaitai.struct.{ImportList, RuntimeConfig}
-import org.scalatest.FunSuite
+import org.scalatest.{FunSuite, Tag}
 import org.scalatest.Matchers._
 
 class TranslatorSpec extends FunSuite {
@@ -226,6 +226,16 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "[255, 0, 255].pack('C*')"
   ))
 
+  full("[0, 1, 2].length", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "std::string(\"\\x00\\x01\\x02\", 3).length()",
+    JavaCompiler -> "new byte[] { 0, 1, 2 }.length",
+    LuaCompiler -> "string.len(\"str\")",
+    PerlCompiler -> "length(pack('C*', (0, 1, 2)))",
+    PHPCompiler -> "strlen(\"\\x00\\x01\\x02\")",
+    PythonCompiler -> "len(b\"\\x00\\x01\\x02\")",
+    RubyCompiler -> "[0, 1, 2].pack('C*').size"
+  ))
+
   full("a[42]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->at(42)",
     CSharpCompiler -> "A[42]",
@@ -457,7 +467,7 @@ class TranslatorSpec extends FunSuite {
 
     langs.foreach { case (langObj, tr) =>
       val langName = LanguageCompilerStatic.CLASS_TO_NAME(langObj)
-      test(s"$langName:$src") {
+      test(s"$langName:$src", Tag(langName), Tag(src)) {
         eo match {
           case Some(e) =>
             val tr: BaseTranslator = langs(langObj)

From 81f4652023b14dcf327d576a78e9708bf9ac3f2d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 31 Mar 2018 13:45:31 +0100
Subject: [PATCH 090/230] Added some bytes length implementations

---
 .../main/scala/io/kaitai/struct/translators/CppTranslator.scala | 2 ++
 .../scala/io/kaitai/struct/translators/JavaTranslator.scala     | 2 ++
 .../main/scala/io/kaitai/struct/translators/PHPTranslator.scala | 2 ++
 .../scala/io/kaitai/struct/translators/PythonTranslator.scala   | 2 ++
 .../scala/io/kaitai/struct/translators/RubyTranslator.scala     | 2 ++
 5 files changed, 10 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 87aa800cc..18cae159a 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -124,6 +124,8 @@ class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends B
   }
   override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String =
     s"${CppCompiler.kstreamName}::bytes_to_str($bytesExpr, ${translate(encoding)})"
+  override def bytesLength(b: Ast.expr): String =
+    s"${translate(b)}.length()"
   override def strLength(s: expr): String =
     s"${translate(s)}.length()"
   override def strReverse(s: expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index c67e25a76..55e3590c3 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -108,6 +108,8 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
     importList.add("java.nio.charset.Charset")
     s"new String($bytesExpr, Charset.forName(${translate(encoding)}))"
   }
+  override def bytesLength(b: Ast.expr): String =
+    s"${translate(b)}.length"
   override def strLength(s: expr): String =
     s"${translate(s)}.length()"
   override def strReverse(s: expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
index 4b0aa9699..c2dbcffeb 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
@@ -95,6 +95,8 @@ class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseT
   }
   override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String =
     s"${PHPCompiler.kstreamName}::bytesToStr($bytesExpr, ${translate(encoding)})"
+  override def bytesLength(b: Ast.expr): String =
+    s"strlen(${translate(b)})"
   override def strLength(s: expr): String =
     s"strlen(${translate(s)})"
   override def strReverse(s: expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
index ef6fa6b63..583935e23 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
@@ -98,6 +98,8 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList) extends B
   }
   override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String =
     s"($bytesExpr).decode(${translate(encoding)})"
+  override def bytesLength(value: Ast.expr): String =
+    s"len(${translate(value)})"
   override def strLength(value: Ast.expr): String =
     s"len(${translate(value)})"
   override def strReverse(value: Ast.expr): String =
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
index 52975cee5..5f6804765 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -60,6 +60,8 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     translate(i) + s".to_s(${translate(base)})"
   override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String =
     s"($bytesExpr).force_encoding(${translate(encoding)})"
+  override def bytesLength(b: Ast.expr): String =
+    s"${translate(b)}.size"
   override def strLength(s: Ast.expr): String =
     s"${translate(s)}.size"
   override def strReverse(s: Ast.expr): String =

From 59a95e965aab74fbc7fc22f3e5baf804da584bc2 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 31 Mar 2018 23:11:40 +0100
Subject: [PATCH 091/230] Added array type casting enforcement + some empty
 literals array tests

---
 .../struct/translators/TranslatorSpec.scala   | 49 +++++++++++++
 .../io/kaitai/struct/datatype/DataType.scala  | 71 ++++++++++---------
 .../struct/translators/BaseTranslator.scala   | 66 +++++++++++++----
 .../struct/translators/TypeDetector.scala     | 17 ++++-
 4 files changed, 151 insertions(+), 52 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 312f31857..e9fc1bb7e 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -447,6 +447,55 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "other.baz"
   ))
 
+  // empty array casting
+  full("[].as<bytes>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "std::string(\"\", 0)",
+    CSharpCompiler -> "new byte[] {  }",
+    JavaCompiler -> "new byte[] {  }",
+    JavaScriptCompiler -> "[]",
+    LuaCompiler -> "\"\"",
+    PerlCompiler -> "pack('C*', ())",
+    PHPCompiler -> "\"\"",
+    PythonCompiler -> "b\"\"",
+    RubyCompiler -> "[].pack('C*')"
+  ))
+
+  full("[].as<u1[]>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "std::string(\"\")",
+    CSharpCompiler -> "new byte[] { }",
+    JavaCompiler -> "new byte[] { }",
+    JavaScriptCompiler -> "[]",
+    LuaCompiler -> "\"\"",
+    PerlCompiler -> "pack('C*', ())",
+    PHPCompiler -> "\"\"",
+    PythonCompiler -> "b\"\"",
+    RubyCompiler -> "[].pack('C*')"
+  ))
+
+  full("[].as<u4[]>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "std::string(\"\")",
+    CSharpCompiler -> "new byte[] {  }",
+    JavaCompiler -> "new byte[] {  }",
+    JavaScriptCompiler -> "[]",
+    LuaCompiler -> "\"\"",
+    PerlCompiler -> "pack('C*', ())",
+    PHPCompiler -> "\"\"",
+    PythonCompiler -> "b\"\"",
+    RubyCompiler -> "[].pack('C*')"
+  ))
+
+  full("[].as<f8[]>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "std::string(\"\", 0)",
+    CSharpCompiler -> "new double[] { }",
+    JavaCompiler -> "new double[] { }",
+    JavaScriptCompiler -> "[]",
+    LuaCompiler -> "[]",
+    PerlCompiler -> "()",
+    PHPCompiler -> "[]",
+    PythonCompiler -> "[]",
+    RubyCompiler -> "[]"
+  ))
+
   def runTest(src: String, tp: TypeProvider, expType: DataType, expOut: ResultMap) {
     var eo: Option[Ast.expr] = None
     test(s"_expr:$src") {
diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index 0919fe70a..0a7bd677e 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -316,41 +316,42 @@ object DataType {
   private val RePureIntType = """([us])(2|4|8)""".r
   private val RePureFloatType = """f(4|8)""".r
 
-  def pureFromString(dto: Option[String]): DataType = {
-    dto match {
-      case None => CalcBytesType
-      case Some(dt) => dt match {
-        case "u1" => Int1Type(false)
-        case "s1" => Int1Type(true)
-        case RePureIntType(signStr, widthStr) =>
-          IntMultiType(
-            signStr match {
-              case "s" => true
-              case "u" => false
-            },
-            widthStr match {
-              case "2" => Width2
-              case "4" => Width4
-              case "8" => Width8
-            },
-            None
-          )
-        case RePureFloatType(widthStr) =>
-          FloatMultiType(
-            widthStr match {
-              case "4" => Width4
-              case "8" => Width8
-            },
-            None
-          )
-        case "str" => CalcStrType
-        case "bool" => CalcBooleanType
-        case "struct" => KaitaiStructType
-        case "io" => KaitaiStreamType
-        case "any" => AnyType
-        case _ => UserTypeInstream(classNameToList(dt), None)
-      }
-    }
+  def pureFromString(dto: Option[String]): DataType = dto match {
+    case None => CalcBytesType
+    case Some(dt) => pureFromString(dt)
+  }
+
+  def pureFromString(dt: String): DataType = dt match {
+    case "bytes" => CalcBytesType
+    case "u1" => Int1Type(false)
+    case "s1" => Int1Type(true)
+    case RePureIntType(signStr, widthStr) =>
+      IntMultiType(
+        signStr match {
+          case "s" => true
+          case "u" => false
+        },
+        widthStr match {
+          case "2" => Width2
+          case "4" => Width4
+          case "8" => Width8
+        },
+        None
+      )
+    case RePureFloatType(widthStr) =>
+      FloatMultiType(
+        widthStr match {
+          case "4" => Width4
+          case "8" => Width8
+        },
+        None
+      )
+    case "str" => CalcStrType
+    case "bool" => CalcBooleanType
+    case "struct" => KaitaiStructType
+    case "io" => KaitaiStreamType
+    case "any" => AnyType
+    case _ => UserTypeInstream(classNameToList(dt), None)
   }
 
   def getEncoding(curEncoding: Option[String], metaDef: MetaSpec, path: List[String]): String = {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 2bd37ceb3..8a25020f0 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -118,24 +118,21 @@ abstract class BaseTranslator(val provider: TypeProvider)
         translateCall(call)
       case Ast.expr.List(values: Seq[Ast.expr]) =>
         val t = detectArrayType(values)
-        t match {
-          case Int1Type(_) =>
-            val literalBytes: Seq[Byte] = values.map {
-              case Ast.expr.IntNum(x) =>
-                if (x < 0 || x > 0xff) {
-                  throw new TypeMismatchError(s"got a weird byte value in byte array: $x")
-                } else {
-                  x.toByte
-                }
-              case n =>
-                throw new TypeMismatchError(s"got $n in byte array, unable to put it literally")
+        doGuessArrayLiteral(t, values)
+      case Ast.expr.CastToType(value, typeName) =>
+        value match {
+          case array: Ast.expr.List =>
+            // Special handling for literal arrays: if cast is present,
+            // then we don't need to guess the data type
+            detectType(v) match {
+              case _: BytesType =>
+                doByteArray(array.elts)
+              case ArrayType(elType) =>
+                doArrayLiteral(elType, array.elts)
             }
-            doByteArrayLiteral(literalBytes)
           case _ =>
-            doArrayLiteral(t, values)
+            doCast(value, typeName)
         }
-      case Ast.expr.CastToType(value, typeName) =>
-        doCast(value, typeName)
     }
   }
 
@@ -143,8 +140,47 @@ abstract class BaseTranslator(val provider: TypeProvider)
   def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String
   def doCast(value: Ast.expr, typeName: Ast.typeId): String = translate(value)
 
+  def doGuessArrayLiteral(elementType: DataType, values: Seq[Ast.expr]): String = elementType match {
+    case Int1Type(_) =>
+      val literalBytes: Seq[Byte] = values.map {
+        case Ast.expr.IntNum(x) =>
+          if (x < 0 || x > 0xff) {
+            throw new TypeMismatchError(s"got a weird byte value in byte array: $x")
+          } else {
+            x.toByte
+          }
+        case n =>
+          throw new TypeMismatchError(s"got $n in byte array, unable to put it literally")
+      }
+      doByteArrayLiteral(literalBytes)
+    case _ =>
+      doArrayLiteral(elementType, values)
+  }
+
+  def valuesAsByteArrayLiteral(elts: Seq[Ast.expr]): Option[Seq[Byte]] = {
+    Some(elts.map {
+      case Ast.expr.IntNum(x) =>
+        if (x < 0 || x > 0xff) {
+          return None
+        } else {
+          x.toByte
+        }
+      case _ =>
+        return None
+    })
+  }
+
+  def doByteArray(elts: Seq[Ast.expr]): String = {
+    valuesAsByteArrayLiteral(elts) match {
+      case Some(arr) =>
+        doByteArrayLiteral(arr)
+      case None =>
+        doByteArrayNonLiteral(elts)
+    }
+  }
   def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = "[" + value.map((v) => translate(v)).mkString(", ") + "]"
   def doByteArrayLiteral(arr: Seq[Byte]): String = "[" + arr.map(_ & 0xff).mkString(", ") + "]"
+  def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = ???
 
   def doLocalName(s: String): String = doName(s)
   def doName(s: String): String
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index 9637aa696..293d94f79 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -115,8 +115,21 @@ class TypeDetector(provider: TypeProvider) {
           case Int1Type(_) => CalcBytesType
           case t => ArrayType(t)
         }
-      case Ast.expr.CastToType(value, typeName) =>
-        provider.resolveType(typeName)
+      case Ast.expr.CastToType(_, typeName) =>
+        if ((!typeName.absolute) && typeName.names.size == 1) {
+          // May be it's a reserved pure data type name?
+          DataType.pureFromString(Some(typeName.names(0))) match {
+            case _: UserType =>
+              // No, it's a user type, let's try to resolve it through provider
+              provider.resolveType(typeName)
+            case other =>
+              // Yes, it is!
+              other
+          }
+        } else {
+          // It's a complex type name, it can be only resolved through provider
+          provider.resolveType(typeName)
+        }
     }
   }
 

From 9152193a4379247b2675842d03ff66132b15c98f Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 1 Apr 2018 02:32:37 +0100
Subject: [PATCH 092/230] Allowed array specifiers `[]` in TYPE_NAME
 expressions

---
 shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala   | 2 +-
 .../main/scala/io/kaitai/struct/exprlang/Expressions.scala  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
index ad27bd840..5e95fe0dd 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
@@ -21,7 +21,7 @@ package io.kaitai.struct.exprlang
   */
 object Ast {
   case class identifier(name: String)
-  case class typeId(absolute: Boolean, names: Seq[String])
+  case class typeId(absolute: Boolean, names: Seq[String], isArray: Boolean = false)
 
   // BoolOp() can use left & right?
   sealed trait expr
diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
index f42194df2..ae5d9f1d9 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
@@ -27,9 +27,9 @@ import fastparse.StringReprOps
 object Expressions {
 
   val NAME: P[Ast.identifier] = Lexical.identifier
-  val TYPE_NAME: P[Ast.typeId] = P("::".!.? ~ NAME.rep(1, "::")).map {
-    case (first, names: Seq[Ast.identifier]) =>
-      Ast.typeId(first.nonEmpty, names.map((el) => el.name))
+  val TYPE_NAME: P[Ast.typeId] = P("::".!.? ~ NAME.rep(1, "::") ~ ("[" ~ "]").!.?).map {
+    case (first, names: Seq[Ast.identifier], arrayStr) =>
+      Ast.typeId(first.nonEmpty, names.map((el) => el.name), arrayStr.nonEmpty)
   }
   val INT_NUMBER = Lexical.integer
   val FLOAT_NUMBER = Lexical.floatnumber

From 6688de32b1ba7cdbdf292b1c5b145442fb9189e3 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 1 Apr 2018 02:33:08 +0100
Subject: [PATCH 093/230] ExpressionSpec: added specs for array specifiers in
 cast type names

---
 .../io/kaitai/struct/exprlang/ExpressionsSpec.scala  | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
index 63bad3f4f..93e4a36ea 100644
--- a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
@@ -243,6 +243,18 @@ class ExpressionsSpec extends FunSpec {
       )
     }
 
+    it("parses foo.as<bar[]>") {
+      Expressions.parse("foo.as<bar[]>") should be (
+        CastToType(Name(identifier("foo")), typeId(false, Seq("bar"), true))
+      )
+    }
+
+    it("parses foo.as<::bar::baz[]>") {
+      Expressions.parse("foo.as<::bar::baz[]>") should be (
+        CastToType(Name(identifier("foo")), typeId(true, Seq("bar", "baz"), true))
+      )
+    }
+
     it("parses foo.as") {
       Expressions.parse("foo.as") should be (Attribute(Name(identifier("foo")),identifier("as")))
     }

From e7f6d06dcc9c49d4642ab3efcef4dddb6f9bba52 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 1 Apr 2018 02:33:41 +0100
Subject: [PATCH 094/230] TypeDetector: take array specs into account

---
 .../io/kaitai/struct/translators/TypeDetector.scala | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index 293d94f79..2a06b2548 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -116,20 +116,27 @@ class TypeDetector(provider: TypeProvider) {
           case t => ArrayType(t)
         }
       case Ast.expr.CastToType(_, typeName) =>
-        if ((!typeName.absolute) && typeName.names.size == 1) {
+        val singleType = if ((!typeName.absolute) && typeName.names.size == 1) {
           // May be it's a reserved pure data type name?
           DataType.pureFromString(Some(typeName.names(0))) match {
             case _: UserType =>
               // No, it's a user type, let's try to resolve it through provider
               provider.resolveType(typeName)
-            case other =>
+            case primitiveType =>
               // Yes, it is!
-              other
+              primitiveType
           }
         } else {
           // It's a complex type name, it can be only resolved through provider
           provider.resolveType(typeName)
         }
+
+        // Wrap it in array type, if needed
+        if (typeName.isArray) {
+          ArrayType(singleType)
+        } else {
+          singleType
+        }
     }
   }
 

From bdc572bf53dea06e0e125b058cb7e726d06f385d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 1 Apr 2018 02:34:31 +0100
Subject: [PATCH 095/230] TranslatorSpec: fix many cast-related tests, add
 non-literal byte array via cast test

---
 .../struct/translators/TranslatorSpec.scala   | 50 +++++++++----------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index e9fc1bb7e..efa738d03 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -34,7 +34,6 @@ class TranslatorSpec extends FunSuite {
 
   everybodyExcept("3 / 2", "(3 / 2)", Map(
     JavaScriptCompiler -> "Math.floor(3 / 2)",
-    LuaCompiler -> "3 / 2",
     PerlCompiler -> "int(3 / 2)",
     PHPCompiler -> "intval(3 / 2)",
     PythonCompiler -> "3 // 2"
@@ -44,7 +43,6 @@ class TranslatorSpec extends FunSuite {
 
   everybodyExcept("(1 + 2) / (7 * 8)", "((1 + 2) / (7 * 8))", Map(
     JavaScriptCompiler -> "Math.floor((1 + 2) / (7 * 8))",
-    LuaCompiler -> "(1 + 2) / (7 * 8)",
     PerlCompiler -> "int((1 + 2) / (7 * 8))",
     PHPCompiler -> "intval((1 + 2) / (7 * 8))",
     PythonCompiler -> "(1 + 2) // (7 * 8)"
@@ -460,42 +458,42 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "[].pack('C*')"
   ))
 
-  full("[].as<u1[]>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
+  full("[].as<u1[]>", CalcIntType, ArrayType(Int1Type(false)), Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\")",
-    CSharpCompiler -> "new byte[] { }",
-    JavaCompiler -> "new byte[] { }",
+    CSharpCompiler -> "new List<byte> {  }",
+    JavaCompiler -> "new ArrayList<Integer>(Arrays.asList())",
     JavaScriptCompiler -> "[]",
-    LuaCompiler -> "\"\"",
-    PerlCompiler -> "pack('C*', ())",
-    PHPCompiler -> "\"\"",
-    PythonCompiler -> "b\"\"",
-    RubyCompiler -> "[].pack('C*')"
-  ))
-
-  full("[].as<u4[]>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
-    CppCompiler -> "std::string(\"\")",
-    CSharpCompiler -> "new byte[] {  }",
-    JavaCompiler -> "new byte[] {  }",
-    JavaScriptCompiler -> "[]",
-    LuaCompiler -> "\"\"",
-    PerlCompiler -> "pack('C*', ())",
-    PHPCompiler -> "\"\"",
-    PythonCompiler -> "b\"\"",
-    RubyCompiler -> "[].pack('C*')"
+    LuaCompiler -> "{}",
+    PerlCompiler -> "()",
+    PHPCompiler -> "[]",
+    PythonCompiler -> "[]",
+    RubyCompiler -> "[]"
   ))
 
-  full("[].as<f8[]>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
+  full("[].as<f8[]>", CalcIntType, ArrayType(FloatMultiType(Width8, None)), Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\", 0)",
-    CSharpCompiler -> "new double[] { }",
-    JavaCompiler -> "new double[] { }",
+    CSharpCompiler -> "new List<double> {  }",
+    JavaCompiler -> "new ArrayList<Double>(Arrays.asList())",
     JavaScriptCompiler -> "[]",
-    LuaCompiler -> "[]",
+    LuaCompiler -> "{}",
     PerlCompiler -> "()",
     PHPCompiler -> "[]",
     PythonCompiler -> "[]",
     RubyCompiler -> "[]"
   ))
 
+  full("[0 + 1, 5].as<bytes>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
+    CppCompiler -> "???",
+    CSharpCompiler -> "new byte[] { 0 + 1, 2 }",
+    JavaCompiler -> "new byte[] { 0 + 1, 2 }",
+    JavaScriptCompiler -> "[0 + 1, 2]",
+    LuaCompiler -> "???",
+    PerlCompiler -> "pack('C*', (0 + 1, 2))",
+    PHPCompiler -> "???",
+    PythonCompiler -> "struct.pack('2b', 0 + 1, 2)",
+    RubyCompiler -> "[0 + 1, 2].pack('C*')"
+  ))
+
   def runTest(src: String, tp: TypeProvider, expType: DataType, expOut: ResultMap) {
     var eo: Option[Ast.expr] = None
     test(s"_expr:$src") {

From 810caebe9d42ceb899fc0c381b729d70a560bfdc Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 1 Apr 2018 02:44:31 +0100
Subject: [PATCH 096/230] Some implementations of doByteArrayNonLiteral + test
 fixes

---
 .../kaitai/struct/translators/TranslatorSpec.scala   | 12 ++++++------
 .../kaitai/struct/translators/JavaTranslator.scala   |  2 ++
 .../kaitai/struct/translators/RubyTranslator.scala   |  2 ++
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index efa738d03..2dd443d62 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -484,14 +484,14 @@ class TranslatorSpec extends FunSuite {
 
   full("[0 + 1, 5].as<bytes>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "???",
-    CSharpCompiler -> "new byte[] { 0 + 1, 2 }",
-    JavaCompiler -> "new byte[] { 0 + 1, 2 }",
-    JavaScriptCompiler -> "[0 + 1, 2]",
+    CSharpCompiler -> "new byte[] { (0 + 1), 5 }",
+    JavaCompiler -> "new byte[] { (0 + 1), 5 }",
+    JavaScriptCompiler -> "[(0 + 1), 5]",
     LuaCompiler -> "???",
-    PerlCompiler -> "pack('C*', (0 + 1, 2))",
+    PerlCompiler -> "pack('C*', ((0 + 1), 5))",
     PHPCompiler -> "???",
-    PythonCompiler -> "struct.pack('2b', 0 + 1, 2)",
-    RubyCompiler -> "[0 + 1, 2].pack('C*')"
+    PythonCompiler -> "struct.pack('2b', (0 + 1), 5)",
+    RubyCompiler -> "[(0 + 1), 5].pack('C*')"
   ))
 
   def runTest(src: String, tp: TypeProvider, expType: DataType, expOut: ResultMap) {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index 55e3590c3..a524a25f5 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -31,6 +31,8 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
 
   override def doByteArrayLiteral(arr: Seq[Byte]): String =
     s"new byte[] { ${arr.mkString(", ")} }"
+  override def doByteArrayNonLiteral(elts: Seq[expr]): String =
+    s"new byte[] { ${elts.map(translate).mkString(", ")} }"
 
   override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = {
     (detectType(left), detectType(right), op) match {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
index 5f6804765..9a92be10c 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -9,6 +9,8 @@ import io.kaitai.struct.languages.RubyCompiler
 class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   override def doByteArrayLiteral(arr: Seq[Byte]): String =
     s"${super.doByteArrayLiteral(arr)}.pack('C*')"
+  override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
+    s"[${elts.map(translate).mkString(", ")}].pack('C*')"
 
   // https://github.com/ruby/ruby/blob/trunk/doc/syntax/literals.rdoc#strings
   // https://github.com/ruby/ruby/blob/trunk/string.c - see "rb_str_inspect"

From 2433afbe0bb2a4bb2249aa7172d0d1a239aa6853 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 2 Apr 2018 18:05:24 +0100
Subject: [PATCH 097/230] More doByteArrayNonLiteral implementations, added
 test to enforce casting to array of integers

---
 .../struct/translators/TranslatorSpec.scala       | 15 ++++++++++++++-
 .../struct/translators/CSharpTranslator.scala     |  2 ++
 .../struct/translators/JavaScriptTranslator.scala |  3 +++
 .../struct/translators/PerlTranslator.scala       |  2 ++
 .../struct/translators/PythonTranslator.scala     |  5 ++++-
 5 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 2dd443d62..5f5d5b50d 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -482,11 +482,12 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "[]"
   ))
 
+  // type enforcement: casting to non-literal byte array
   full("[0 + 1, 5].as<bytes>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "???",
     CSharpCompiler -> "new byte[] { (0 + 1), 5 }",
     JavaCompiler -> "new byte[] { (0 + 1), 5 }",
-    JavaScriptCompiler -> "[(0 + 1), 5]",
+    JavaScriptCompiler -> "new Uint8Array([(0 + 1), 5])",
     LuaCompiler -> "???",
     PerlCompiler -> "pack('C*', ((0 + 1), 5))",
     PHPCompiler -> "???",
@@ -494,6 +495,18 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "[(0 + 1), 5].pack('C*')"
   ))
 
+  // type enforcement: casting to array of integers
+  full("[0, 1, 2].as<u1[]>", CalcIntType, ArrayType(Int1Type(false)), Map[LanguageCompilerStatic, String](
+    CSharpCompiler -> "new List<byte> { 0, 1, 2 }",
+    JavaCompiler -> "new ArrayList<Integer>(Arrays.asList(0, 1, 2))",
+    JavaScriptCompiler -> "[0, 1, 2]",
+    LuaCompiler -> "{0, 1, 2}",
+    PerlCompiler -> "(0, 1, 2)",
+    PHPCompiler -> "[0, 1, 2]",
+    PythonCompiler -> "[0, 1, 2]",
+    RubyCompiler -> "[0, 1, 2]"
+  ))
+
   def runTest(src: String, tp: TypeProvider, expType: DataType, expOut: ResultMap) {
     var eo: Option[Ast.expr] = None
     test(s"_expr:$src") {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
index db3ab9657..fea6fc5bc 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
@@ -17,6 +17,8 @@ class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends B
 
   override def doByteArrayLiteral(arr: Seq[Byte]): String =
     s"new byte[] { ${arr.map(_ & 0xff).mkString(", ")} }"
+  override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
+    s"new byte[] { ${elts.map(translate).mkString(", ")} }"
 
   override val asciiCharQuoteMap: Map[Char, String] = Map(
     '\t' -> "\\t",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
index 3605adea4..c938009e3 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaScriptTranslator.scala
@@ -8,6 +8,9 @@ import io.kaitai.struct.format.Identifier
 import io.kaitai.struct.languages.JavaScriptCompiler
 
 class JavaScriptTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
+  override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
+    s"new Uint8Array([${elts.map(translate).mkString(", ")}])"
+
   /**
     * JavaScript rendition of common control character that would use hex form,
     * not octal. "Octal" control character string literals might be accepted
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
index cc6c30ef7..92eb49cfd 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PerlTranslator.scala
@@ -49,6 +49,8 @@ class PerlTranslator(provider: TypeProvider, importList: ImportList) extends Bas
 
   override def doByteArrayLiteral(arr: Seq[Byte]): String =
     s"pack('C*', (${arr.map(_ & 0xff).mkString(", ")}))"
+  override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
+    s"pack('C*', (${elts.map(translate).mkString(", ")}))"
 
   override def anyField(value: Ast.expr, attrName: String): String =
     s"${translate(value)}->${doName(attrName)}"
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
index 583935e23..7b48c2908 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
@@ -36,8 +36,11 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList) extends B
     '\b' -> "\\b"
   )
 
-  override def doByteArrayLiteral(arr: Seq[Byte]): String = {
+  override def doByteArrayLiteral(arr: Seq[Byte]): String =
     "b\"" + Utils.hexEscapeByteArray(arr) + "\""
+  override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = {
+    importList.add("import struct")
+    s"struct.pack('${elts.length}b', ${elts.map(translate).mkString(", ")})"
   }
 
   override def doLocalName(s: String) = {

From 32349bb0949b70ff1dc78d33960a880b96c18bd7 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 2 Apr 2018 21:37:59 +0100
Subject: [PATCH 098/230] Fixed more tests

---
 .../io/kaitai/struct/translators/TranslatorSpec.scala     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 5f5d5b50d..26a018f79 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -208,7 +208,7 @@ class TranslatorSpec extends FunSuite {
     LuaCompiler -> "\"\\034\\000\\010\\064\\065\\066\\092\"",
     PerlCompiler -> "pack('C*', (34, 0, 10, 64, 65, 66, 92))",
     PHPCompiler -> "\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\"",
-    PythonCompiler -> "struct.pack('7b', 34, 0, 10, 64, 65, 66, 92)",
+    PythonCompiler -> "b\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\"",
     RubyCompiler -> "[34, 0, 10, 64, 65, 66, 92].pack('C*')"
   ))
 
@@ -220,7 +220,7 @@ class TranslatorSpec extends FunSuite {
     LuaCompiler -> "\"\\255\\000\\255\"",
     PerlCompiler -> "pack('C*', (255, 0, 255))",
     PHPCompiler -> "\"\\xFF\\x00\\xFF\"",
-    PythonCompiler -> "struct.pack('3b', -1, 0, -1)",
+    PythonCompiler -> "b\"\\255\\000\\255\"",
     RubyCompiler -> "[255, 0, 255].pack('C*')"
   ))
 
@@ -376,7 +376,7 @@ class TranslatorSpec extends FunSuite {
     CppCompiler -> "std::string(\"str\").length()",
     CSharpCompiler -> "\"str\".Length",
     JavaCompiler -> "\"str\".length()",
-    JavaScriptCompiler -> "#\"str\"",
+    JavaScriptCompiler -> "\"str\".length",
     LuaCompiler -> "string.len(\"str\")",
     PerlCompiler -> "length(\"str\")",
     PHPCompiler -> "strlen(\"str\")",
@@ -490,7 +490,7 @@ class TranslatorSpec extends FunSuite {
     JavaScriptCompiler -> "new Uint8Array([(0 + 1), 5])",
     LuaCompiler -> "???",
     PerlCompiler -> "pack('C*', ((0 + 1), 5))",
-    PHPCompiler -> "???",
+    PHPCompiler -> "pack('C*', (0 + 1), 5)",
     PythonCompiler -> "struct.pack('2b', (0 + 1), 5)",
     RubyCompiler -> "[(0 + 1), 5].pack('C*')"
   ))

From 409c13a797212bfd9f57ca75e3165d4ae74cdcfe Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 2 Apr 2018 21:38:13 +0100
Subject: [PATCH 099/230] PHPTranslator: implemented doByteArrayNonLiteral

---
 .../main/scala/io/kaitai/struct/translators/PHPTranslator.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
index c2dbcffeb..cfd59a8e9 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PHPTranslator.scala
@@ -10,6 +10,8 @@ import io.kaitai.struct.{RuntimeConfig, Utils}
 class PHPTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) {
   override def doByteArrayLiteral(arr: Seq[Byte]): String =
     "\"" + Utils.hexEscapeByteArray(arr) + "\""
+  override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
+    s"pack('C*', ${elts.map(translate).mkString(", ")})"
 
   // http://php.net/manual/en/language.types.string.php#language.types.string.syntax.double
   override val asciiCharQuoteMap: Map[Char, String] = Map(

From f1e6055f3ccc437df753f0140d57037b0150a116 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 4 Apr 2018 08:18:39 +0100
Subject: [PATCH 100/230] Added method to test go expressions + some specs

---
 .../struct/translators/TranslatorSpec.scala   | 24 +++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 26a018f79..e30fdfb08 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -6,7 +6,7 @@ import io.kaitai.struct.exprlang.{Ast, Expressions}
 import io.kaitai.struct.format.ClassSpec
 import io.kaitai.struct.languages._
 import io.kaitai.struct.languages.components.LanguageCompilerStatic
-import io.kaitai.struct.{ImportList, RuntimeConfig}
+import io.kaitai.struct.{ImportList, RuntimeConfig, StringLanguageOutputWriter}
 import org.scalatest.{FunSuite, Tag}
 import org.scalatest.Matchers._
 
@@ -82,6 +82,7 @@ class TranslatorSpec extends FunSuite {
   full("true", CalcBooleanType, CalcBooleanType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "true",
     CSharpCompiler -> "true",
+    GoCompiler -> "true",
     JavaCompiler -> "true",
     JavaScriptCompiler -> "true",
     LuaCompiler -> "true",
@@ -94,6 +95,7 @@ class TranslatorSpec extends FunSuite {
   full("false", CalcBooleanType, CalcBooleanType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "false",
     CSharpCompiler -> "false",
+    GoCompiler -> "false",
     JavaCompiler -> "false",
     JavaScriptCompiler -> "false",
     LuaCompiler -> "false",
@@ -513,9 +515,12 @@ class TranslatorSpec extends FunSuite {
       eo = Some(Expressions.parse(src))
     }
 
-    val langs = Map[LanguageCompilerStatic, BaseTranslator](
+    val goOutput = new StringLanguageOutputWriter("\t")
+
+    val langs = Map[LanguageCompilerStatic, AbstractTranslator with TypeDetector](
       CppCompiler -> new CppTranslator(tp, new ImportList()),
       CSharpCompiler -> new CSharpTranslator(tp, new ImportList()),
+      GoCompiler -> new GoTranslator(goOutput, tp, new ImportList()),
       JavaCompiler -> new JavaTranslator(tp, new ImportList()),
       JavaScriptCompiler -> new JavaScriptTranslator(tp),
       LuaCompiler -> new LuaTranslator(tp, new ImportList()),
@@ -530,11 +535,22 @@ class TranslatorSpec extends FunSuite {
       test(s"$langName:$src", Tag(langName), Tag(src)) {
         eo match {
           case Some(e) =>
-            val tr: BaseTranslator = langs(langObj)
+            val tr: AbstractTranslator with TypeDetector = langs(langObj)
             expOut.get(langObj) match {
               case Some(expResult) =>
                 tr.detectType(e) should be(expType)
-                tr.translate(e) should be(expResult)
+                val actResult1 = tr.translate(e)
+                val actResult2 = langObj match {
+                  case GoCompiler =>
+                    val preExpr = goOutput.result
+                    if (preExpr.isEmpty) {
+                      actResult1
+                    } else {
+                      preExpr + "\n" + actResult1
+                    }
+                  case _ => actResult1
+                }
+                actResult2 should be(expResult)
               case None =>
                 fail("no expected result")
             }

From 7f2d0ceb328c5e9b6ed631d24c7c69410f064d61 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 4 Apr 2018 08:40:43 +0100
Subject: [PATCH 101/230] GoTranslator: implemented UnaryOp, IfExp, Compare

---
 .../struct/translators/GoTranslator.scala     | 49 +++++++++++++++++--
 1 file changed, 46 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index b4be487eb..b7faa7b98 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -48,9 +48,24 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
           case (ltype, rtype, _) =>
             throw new TypeMismatchError(s"can't do $ltype $op $rtype")
         }
-//      case Ast.expr.UnaryOp(op, operand) =>
-//      case Ast.expr.IfExp(condition, ifTrue, ifFalse) =>
-//      case Ast.expr.Compare(left, ops, right) =>
+      case Ast.expr.UnaryOp(op, operand) =>
+        ResultString(unaryOp(op) + (operand match {
+          case Ast.expr.IntNum(_) | Ast.expr.FloatNum(_) =>
+            translate(operand)
+          case _ =>
+            s"(${translate(operand)})"
+        }))
+      case Ast.expr.IfExp(condition, ifTrue, ifFalse) =>
+        trIfExp(condition, ifTrue, ifFalse)
+      case Ast.expr.Compare(left, op, right) =>
+        (detectType(left), detectType(right)) match {
+          case (_: NumericType, _: NumericType) =>
+            trNumericCompareOp(left, op, right)
+          case (_: StrType, _: StrType) =>
+            trStrCompareOp(left, op, right)
+          case (ltype, rtype) =>
+            throw new TypeMismatchError(s"can't do $ltype $op $rtype")
+        }
 //      case Ast.expr.EnumByLabel(enumName, label) =>
 //      case Ast.expr.EnumById(enumName, id) =>
 //      case Ast.expr.CastToType(value, typeName) =>
@@ -76,6 +91,12 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   def trStrConcat(left: Ast.expr, right: Ast.expr): TranslatorResult =
     ResultString(translate(left) + " + " + translate(right))
 
+  def trNumericCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult =
+    ResultString(doNumericCompareOp(left, op, right))
+
+  def trStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult =
+    ResultString(doStrCompareOp(left, op, right))
+
 //  override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = {
 //    val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t)
 //    val commaStr = value.map((v) => translate(v)).mkString(", ")
@@ -85,6 +106,12 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
 //  override def doByteArrayLiteral(arr: Seq[Byte]): String =
 //    s"new byte[] { ${arr.mkString(", ")} }"
 
+  override def unaryOp(op: Ast.unaryop): String = op match {
+    case Ast.unaryop.Invert => "^"
+    case Ast.unaryop.Minus => "-"
+    case Ast.unaryop.Not => "!"
+  }
+
   override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = {
     (detectType(left), detectType(right), op) match {
       case (_: IntType, _: IntType, Ast.operator.Mod) =>
@@ -124,6 +151,22 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
       "_buf"
   }
 
+  def trIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): ResultLocalVar = {
+    val v1 = allocateLocalVar()
+    val typ = detectType(ifTrue)
+    out.puts(s"var ${localVarName(v1)} ${GoCompiler.kaitaiType2NativeType(typ)};")
+    out.puts(s"if (${translate(condition)}) {")
+    out.inc
+    out.puts(s"${localVarName(v1)} = ${translate(ifTrue)}")
+    out.dec
+    out.puts("} else {")
+    out.inc
+    out.puts(s"${localVarName(v1)} = ${translate(ifFalse)}")
+    out.dec
+    out.puts("}")
+    ResultLocalVar(v1)
+  }
+
 //  override def doEnumByLabel(enumTypeAbs: List[String], label: String): String =
 //    s"${enumClass(enumTypeAbs)}.${label.toUpperCase}"
 //  override def doEnumById(enumTypeAbs: List[String], id: String): String =

From f4f040058fbd914ef0d916cd6ecc3e1e18d0a338 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 4 Apr 2018 08:41:19 +0100
Subject: [PATCH 102/230] TranslatorSpec: fixed go result rendering, added
 ternary operator conversion result

---
 .../struct/translators/TranslatorSpec.scala     | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index e30fdfb08..b7bc67ed0 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -55,6 +55,13 @@ class TranslatorSpec extends FunSuite {
   full("2 < 3 ? \"foo\" : \"bar\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "(2 < 3) ? (std::string(\"foo\")) : (std::string(\"bar\"))",
     CSharpCompiler -> "2 < 3 ? \"foo\" : \"bar\"",
+    GoCompiler -> """var tmp1 string;
+                    |if (2 < 3) {
+                    |  tmp1 = "foo"
+                    |} else {
+                    |  tmp1 = "bar"
+                    |}
+                    |tmp1""".stripMargin,
     JavaCompiler -> "2 < 3 ? \"foo\" : \"bar\"",
     JavaScriptCompiler -> "2 < 3 ? \"foo\" : \"bar\"",
     LuaCompiler -> "2 < 3 and \"foo\" or \"bar\"",
@@ -515,7 +522,7 @@ class TranslatorSpec extends FunSuite {
       eo = Some(Expressions.parse(src))
     }
 
-    val goOutput = new StringLanguageOutputWriter("\t")
+    val goOutput = new StringLanguageOutputWriter("  ")
 
     val langs = Map[LanguageCompilerStatic, AbstractTranslator with TypeDetector](
       CppCompiler -> new CppTranslator(tp, new ImportList()),
@@ -541,13 +548,7 @@ class TranslatorSpec extends FunSuite {
                 tr.detectType(e) should be(expType)
                 val actResult1 = tr.translate(e)
                 val actResult2 = langObj match {
-                  case GoCompiler =>
-                    val preExpr = goOutput.result
-                    if (preExpr.isEmpty) {
-                      actResult1
-                    } else {
-                      preExpr + "\n" + actResult1
-                    }
+                  case GoCompiler => goOutput.result + actResult1
                   case _ => actResult1
                 }
                 actResult2 should be(expResult)

From b03749117ac452a88d2013d81813851c7dca179d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 4 Apr 2018 09:03:45 +0100
Subject: [PATCH 103/230] GoTranslator: just use default string comparison

---
 .../io/kaitai/struct/translators/GoTranslator.scala    | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index b7faa7b98..a032ee6a8 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -177,16 +177,6 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     enumTypeRel.map((x) => Utils.upperCamelCase(x)).mkString(".")
   }
 
-  override def doStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr) = {
-    if (op == Ast.cmpop.Eq) {
-      s"${translate(left)}.equals(${translate(right)})"
-    } else if (op == Ast.cmpop.NotEq) {
-      s"!(${translate(left)}).equals(${translate(right)})"
-    } else {
-      s"(${translate(left)}.compareTo(${translate(right)}) ${cmpOp(op)} 0)"
-    }
-  }
-
   override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = {
     op match {
       case Ast.cmpop.Eq =>

From 14f69712eebb40dda96edcbbbe2ffac743f7b565 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 4 Apr 2018 09:06:21 +0100
Subject: [PATCH 104/230] More expected Go translation results

---
 .../kaitai/struct/translators/TranslatorSpec.scala | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index b7bc67ed0..4d25e4305 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -71,8 +71,12 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "2 < 3 ? \"foo\" : \"bar\""
   ))
 
-  everybody("~777", "~777")
-  everybody("~(7+3)", "~((7 + 3))")
+  everybodyExcept("~777", "~777", Map[LanguageCompilerStatic, String](
+    GoCompiler -> "^777"
+  ))
+  everybodyExcept("~(7+3)", "~((7 + 3))", Map[LanguageCompilerStatic, String](
+    GoCompiler -> "^((7 + 3))"
+  ))
 
   // Simple float operations
   everybody("1.2 + 3.4", "(1.2 + 3.4)", CalcFloatType)
@@ -212,6 +216,7 @@ class TranslatorSpec extends FunSuite {
   full("[34, 0, 10, 64, 65, 66, 92]", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\", 7)",
     CSharpCompiler -> "new byte[] { 34, 0, 10, 64, 65, 66, 92 }",
+    GoCompiler -> "\"\\x22\\x00\\x0A\\x40\\x41\\x42\\x5C\"",
     JavaCompiler -> "new byte[] { 34, 0, 10, 64, 65, 66, 92 }",
     JavaScriptCompiler -> "[34, 0, 10, 64, 65, 66, 92]",
     LuaCompiler -> "\"\\034\\000\\010\\064\\065\\066\\092\"",
@@ -224,6 +229,7 @@ class TranslatorSpec extends FunSuite {
   full("[255, 0, 255]", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\\xFF\\x00\\xFF\", 3)",
     CSharpCompiler -> "new byte[] { 255, 0, 255 }",
+    GoCompiler -> "\"\\xFF\\x00\\xFF\"",
     JavaCompiler -> "new byte[] { -1, 0, -1 }",
     JavaScriptCompiler -> "[255, 0, 255]",
     LuaCompiler -> "\"\\255\\000\\255\"",
@@ -235,6 +241,7 @@ class TranslatorSpec extends FunSuite {
 
   full("[0, 1, 2].length", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\\x00\\x01\\x02\", 3).length()",
+    GoCompiler -> "len(\"\\x00\\x01\\x02\")",
     JavaCompiler -> "new byte[] { 0, 1, 2 }.length",
     LuaCompiler -> "string.len(\"str\")",
     PerlCompiler -> "length(pack('C*', (0, 1, 2)))",
@@ -303,6 +310,7 @@ class TranslatorSpec extends FunSuite {
   full("\"str\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"str\")",
     CSharpCompiler -> "\"str\"",
+    GoCompiler -> "\"str\"",
     JavaCompiler -> "\"str\"",
     JavaScriptCompiler -> "\"str\"",
     LuaCompiler -> "\"str\"",
@@ -315,6 +323,7 @@ class TranslatorSpec extends FunSuite {
   full("\"str\\nnext\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"str\\nnext\")",
     CSharpCompiler -> "\"str\\nnext\"",
+    GoCompiler -> "\"str\\nnext\"",
     JavaCompiler -> "\"str\\nnext\"",
     JavaScriptCompiler -> "\"str\\nnext\"",
     LuaCompiler -> "\"str\\nnext\"",
@@ -384,6 +393,7 @@ class TranslatorSpec extends FunSuite {
   full("\"str\".length", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"str\").length()",
     CSharpCompiler -> "\"str\".Length",
+    GoCompiler -> "utf8.RuneCountInString(\"str\")",
     JavaCompiler -> "\"str\".length()",
     JavaScriptCompiler -> "\"str\".length",
     LuaCompiler -> "string.len(\"str\")",

From 9fd227ef7517d674acd6308e078b8deb35a30340 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 4 Apr 2018 10:09:06 +0100
Subject: [PATCH 105/230] Extracted common arrays and cast logic into
 CommonArraysAndCast

---
 .../struct/translators/BaseTranslator.scala   | 61 ++----------
 .../translators/CommonArraysAndCast.scala     | 99 +++++++++++++++++++
 .../struct/translators/TypeDetector.scala     | 52 ++++++----
 3 files changed, 136 insertions(+), 76 deletions(-)
 create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 8a25020f0..66e799359 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -16,7 +16,8 @@ import io.kaitai.struct.precompile.TypeMismatchError
   *
   * Given that there are many of these abstract methods, to make it more
   * maintainable, they are grouped into several abstract traits:
-  * [[CommonLiterals]], [[CommonOps]].
+  * [[CommonLiterals]], [[CommonOps]], [[CommonMethods]],
+  * [[CommonArraysAndCast]].
   *
   * This translator implementation also handles user-defined types and
   * fields properly - it uses given [[TypeProvider]] to resolve these.
@@ -28,6 +29,7 @@ abstract class BaseTranslator(val provider: TypeProvider)
   with AbstractTranslator
   with CommonLiterals
   with CommonOps
+  with CommonArraysAndCast[String]
   with CommonMethods[String] {
 
   /**
@@ -117,22 +119,9 @@ abstract class BaseTranslator(val provider: TypeProvider)
       case call: Ast.expr.Call =>
         translateCall(call)
       case Ast.expr.List(values: Seq[Ast.expr]) =>
-        val t = detectArrayType(values)
-        doGuessArrayLiteral(t, values)
-      case Ast.expr.CastToType(value, typeName) =>
-        value match {
-          case array: Ast.expr.List =>
-            // Special handling for literal arrays: if cast is present,
-            // then we don't need to guess the data type
-            detectType(v) match {
-              case _: BytesType =>
-                doByteArray(array.elts)
-              case ArrayType(elType) =>
-                doArrayLiteral(elType, array.elts)
-            }
-          case _ =>
-            doCast(value, typeName)
-        }
+        doGuessArrayLiteral(values)
+      case ctt: Ast.expr.CastToType =>
+        doCastOrArray(ctt)
     }
   }
 
@@ -140,44 +129,6 @@ abstract class BaseTranslator(val provider: TypeProvider)
   def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String
   def doCast(value: Ast.expr, typeName: Ast.typeId): String = translate(value)
 
-  def doGuessArrayLiteral(elementType: DataType, values: Seq[Ast.expr]): String = elementType match {
-    case Int1Type(_) =>
-      val literalBytes: Seq[Byte] = values.map {
-        case Ast.expr.IntNum(x) =>
-          if (x < 0 || x > 0xff) {
-            throw new TypeMismatchError(s"got a weird byte value in byte array: $x")
-          } else {
-            x.toByte
-          }
-        case n =>
-          throw new TypeMismatchError(s"got $n in byte array, unable to put it literally")
-      }
-      doByteArrayLiteral(literalBytes)
-    case _ =>
-      doArrayLiteral(elementType, values)
-  }
-
-  def valuesAsByteArrayLiteral(elts: Seq[Ast.expr]): Option[Seq[Byte]] = {
-    Some(elts.map {
-      case Ast.expr.IntNum(x) =>
-        if (x < 0 || x > 0xff) {
-          return None
-        } else {
-          x.toByte
-        }
-      case _ =>
-        return None
-    })
-  }
-
-  def doByteArray(elts: Seq[Ast.expr]): String = {
-    valuesAsByteArrayLiteral(elts) match {
-      case Some(arr) =>
-        doByteArrayLiteral(arr)
-      case None =>
-        doByteArrayNonLiteral(elts)
-    }
-  }
   def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = "[" + value.map((v) => translate(v)).mkString(", ") + "]"
   def doByteArrayLiteral(arr: Seq[Byte]): String = "[" + arr.map(_ & 0xff).mkString(", ") + "]"
   def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String = ???
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala
new file mode 100644
index 000000000..900c7b23f
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala
@@ -0,0 +1,99 @@
+package io.kaitai.struct.translators
+
+import io.kaitai.struct.datatype.DataType
+import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.exprlang.Ast
+import io.kaitai.struct.precompile.TypeMismatchError
+
+/**
+  * Common implementation of arrays translations:
+  *
+  * * type guessing
+  * * type enforcing with a cast
+  * * rendering of byte arrays and true arrays
+  * * call to actual casting implementation
+  *
+  * @tparam T translation result type
+  */
+trait CommonArraysAndCast[T] extends TypeDetector {
+  /**
+    * Processes elements inside a given [[Ast.expr.List]] element to render them
+    * as either byte array literal or true array.
+    * @param values elements from a list
+    * @return translation result
+    */
+  def doGuessArrayLiteral(values: Seq[Ast.expr]): T = {
+    val elementType = detectArrayType(values)
+    elementType match {
+      case Int1Type(_) =>
+        val literalBytes: Seq[Byte] = values.map {
+          case Ast.expr.IntNum(x) =>
+            if (x < 0 || x > 0xff) {
+              throw new TypeMismatchError(s"got a weird byte value in byte array: $x")
+            } else {
+              x.toByte
+            }
+          case n =>
+            throw new TypeMismatchError(s"got $n in byte array, unable to put it literally")
+        }
+        doByteArrayLiteral(literalBytes)
+      case _ =>
+        doArrayLiteral(elementType, values)
+    }
+  }
+
+  /**
+    * Processes an [[Ast.expr.CastToType]] element, by checking if
+    * this is an literal array type enforcement cast first, and
+    * rendering it accordingly as proper literal, or invoking
+    * the normal [[doCast]] otherwise.
+    * @param v CastToType element
+    * @return translation result
+    */
+  def doCastOrArray(v: Ast.expr.CastToType): T = {
+    v.value match {
+      case array: Ast.expr.List =>
+        // Special handling for literal arrays: if cast is present,
+        // then we don't need to guess the data type
+        detectCastType(v.typeName) match {
+          case _: BytesType =>
+            doByteArray(array.elts)
+          case ArrayType(elType) =>
+            doArrayLiteral(elType, array.elts)
+          case _ =>
+            // No luck, this is some kind of weird cast, not a type enforcement;
+            // Just do it and let real type casting deal with it.
+            doCast(v.value, v.typeName)
+        }
+      case _ =>
+        doCast(v.value, v.typeName)
+    }
+  }
+
+  def doCast(value: Ast.expr, typeName: Ast.typeId): T
+  def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): T
+  def doByteArrayLiteral(arr: Seq[Byte]): T
+  def doByteArrayNonLiteral(elts: Seq[Ast.expr]): T
+
+  private def doByteArray(elts: Seq[Ast.expr]): T = {
+    valuesAsByteArrayLiteral(elts) match {
+      case Some(arr) =>
+        doByteArrayLiteral(arr)
+      case None =>
+        doByteArrayNonLiteral(elts)
+    }
+  }
+
+  private def valuesAsByteArrayLiteral(elts: Seq[Ast.expr]): Option[Seq[Byte]] = {
+    Some(elts.map {
+      case Ast.expr.IntNum(x) =>
+        if (x < 0 || x > 0xff) {
+          return None
+        } else {
+          x.toByte
+        }
+      case _ =>
+        return None
+    })
+  }
+}
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index 2a06b2548..f4ed11053 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -116,32 +116,13 @@ class TypeDetector(provider: TypeProvider) {
           case t => ArrayType(t)
         }
       case Ast.expr.CastToType(_, typeName) =>
-        val singleType = if ((!typeName.absolute) && typeName.names.size == 1) {
-          // May be it's a reserved pure data type name?
-          DataType.pureFromString(Some(typeName.names(0))) match {
-            case _: UserType =>
-              // No, it's a user type, let's try to resolve it through provider
-              provider.resolveType(typeName)
-            case primitiveType =>
-              // Yes, it is!
-              primitiveType
-          }
-        } else {
-          // It's a complex type name, it can be only resolved through provider
-          provider.resolveType(typeName)
-        }
-
-        // Wrap it in array type, if needed
-        if (typeName.isArray) {
-          ArrayType(singleType)
-        } else {
-          singleType
-        }
+        detectCastType(typeName)
     }
   }
 
   /**
     * Detects resulting data type of a given attribute expression.
+    *
     * @note Must be kept in sync with [[CommonMethods.translateAttribute]]
     * @param value value part of attribute expression
     * @param attr attribute identifier part of attribute expression
@@ -252,6 +233,35 @@ class TypeDetector(provider: TypeProvider) {
       case Some(t) => t
     }
   }
+
+  /**
+    * Detects cast type determined by a typeId definition.
+    * @param typeName typeId definition to use
+    * @return data type
+    */
+  def detectCastType(typeName: Ast.typeId): DataType = {
+    val singleType = if ((!typeName.absolute) && typeName.names.size == 1) {
+      // May be it's a reserved pure data type name?
+      DataType.pureFromString(Some(typeName.names(0))) match {
+        case _: UserType =>
+          // No, it's a user type, let's try to resolve it through provider
+          provider.resolveType(typeName)
+        case primitiveType =>
+          // Yes, it is!
+          primitiveType
+      }
+    } else {
+      // It's a complex type name, it can be only resolved through provider
+      provider.resolveType(typeName)
+    }
+
+    // Wrap it in array type, if needed
+    if (typeName.isArray) {
+      ArrayType(singleType)
+    } else {
+      singleType
+    }
+  }
 }
 
 object TypeDetector {

From 2e00d2bcc46b8957fad9beb9ed2f5fcf6884de25 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 4 Apr 2018 10:25:54 +0100
Subject: [PATCH 106/230] GoTranslator: added arrays rendering

---
 .../struct/translators/GoTranslator.scala     | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index a032ee6a8..7a38c54b2 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -1,5 +1,6 @@
 package io.kaitai.struct.translators
 
+import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.{ClassSpec, Identifier}
@@ -16,6 +17,7 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   with AbstractTranslator
   with CommonLiterals
   with CommonOps
+  with CommonArraysAndCast[TranslatorResult]
   with CommonMethods[TranslatorResult] {
 
   var returnRes: Option[String] = None
@@ -68,11 +70,13 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
         }
 //      case Ast.expr.EnumByLabel(enumName, label) =>
 //      case Ast.expr.EnumById(enumName, id) =>
-//      case Ast.expr.CastToType(value, typeName) =>
+      case ctt: Ast.expr.CastToType =>
+        doCastOrArray(ctt)
 //      case Ast.expr.Subscript(value, idx) =>
       case Ast.expr.Name(name: Ast.identifier) =>
         trLocalName(name.name)
-//      case Ast.expr.List(elts) =>
+      case Ast.expr.List(elts) =>
+        doGuessArrayLiteral(elts)
       case call: Ast.expr.Attribute =>
         translateAttribute(call)
       case call: Ast.expr.Call =>
@@ -195,6 +199,17 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
 //  override def doCast(value: Ast.expr, typeName: String): String =
 //    s"((${Utils.upperCamelCase(typeName)}) (${translate(value)}))"
 
+  override def doCast(value: Ast.expr, typeName: Ast.typeId): TranslatorResult = ???
+
+  override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]) =
+    ResultString(s"[]${GoCompiler.kaitaiType2NativeType(t)}{${value.map(translate).mkString(", ")}}")
+
+  override def doByteArrayLiteral(arr: Seq[Byte]): TranslatorResult =
+    ResultString("\"" + Utils.hexEscapeByteArray(arr) + "\"")
+
+  override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): TranslatorResult =
+    ResultString("string([]byte{" + elts.map(translate).mkString(", ") + "})")
+
   // Predefined methods of various types
 //  override def strToInt(s: Ast.expr, base: Ast.expr): String =
 //    s"Long.parseLong(${translate(s)}, ${translate(base)})"

From f33d326050073055c1fe6791c57b911989a09463 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 4 Apr 2018 10:26:18 +0100
Subject: [PATCH 107/230] TranslatorSpec: added Go renditions of various arrays

---
 .../scala/io/kaitai/struct/translators/TranslatorSpec.scala  | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 4d25e4305..fbfdd3cd3 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -468,6 +468,7 @@ class TranslatorSpec extends FunSuite {
   full("[].as<bytes>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\", 0)",
     CSharpCompiler -> "new byte[] {  }",
+    GoCompiler -> "\"\"",
     JavaCompiler -> "new byte[] {  }",
     JavaScriptCompiler -> "[]",
     LuaCompiler -> "\"\"",
@@ -480,6 +481,7 @@ class TranslatorSpec extends FunSuite {
   full("[].as<u1[]>", CalcIntType, ArrayType(Int1Type(false)), Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\")",
     CSharpCompiler -> "new List<byte> {  }",
+    GoCompiler -> "[]uint8{}",
     JavaCompiler -> "new ArrayList<Integer>(Arrays.asList())",
     JavaScriptCompiler -> "[]",
     LuaCompiler -> "{}",
@@ -492,6 +494,7 @@ class TranslatorSpec extends FunSuite {
   full("[].as<f8[]>", CalcIntType, ArrayType(FloatMultiType(Width8, None)), Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\", 0)",
     CSharpCompiler -> "new List<double> {  }",
+    GoCompiler -> "[]float64{}",
     JavaCompiler -> "new ArrayList<Double>(Arrays.asList())",
     JavaScriptCompiler -> "[]",
     LuaCompiler -> "{}",
@@ -505,6 +508,7 @@ class TranslatorSpec extends FunSuite {
   full("[0 + 1, 5].as<bytes>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "???",
     CSharpCompiler -> "new byte[] { (0 + 1), 5 }",
+    GoCompiler -> "string([]byte{(0 + 1), 5})",
     JavaCompiler -> "new byte[] { (0 + 1), 5 }",
     JavaScriptCompiler -> "new Uint8Array([(0 + 1), 5])",
     LuaCompiler -> "???",
@@ -517,6 +521,7 @@ class TranslatorSpec extends FunSuite {
   // type enforcement: casting to array of integers
   full("[0, 1, 2].as<u1[]>", CalcIntType, ArrayType(Int1Type(false)), Map[LanguageCompilerStatic, String](
     CSharpCompiler -> "new List<byte> { 0, 1, 2 }",
+    GoCompiler -> "[]uint8{0, 1, 2}",
     JavaCompiler -> "new ArrayList<Integer>(Arrays.asList(0, 1, 2))",
     JavaScriptCompiler -> "[0, 1, 2]",
     LuaCompiler -> "{0, 1, 2}",

From 781538a19ce0bbcb828b7d03ddc6ea9686319d2a Mon Sep 17 00:00:00 2001
From: cugu <git@cugu.eu>
Date: Wed, 4 Apr 2018 23:42:29 +0200
Subject: [PATCH 108/230] Complete Expressions tests for go

---
 .../struct/translators/TranslatorSpec.scala   | 20 +++++++++++++++
 .../struct/translators/GoTranslator.scala     | 25 +++++++------------
 2 files changed, 29 insertions(+), 16 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index fbfdd3cd3..9f8d09e2a 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -119,6 +119,7 @@ class TranslatorSpec extends FunSuite {
   full("some_bool.to_i", CalcBooleanType, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "some_bool()",
     CSharpCompiler -> "(SomeBool ? 1 : 0)",
+    GoCompiler -> "func()(int) { if ( this.SomeBool ) { return 1 }; return 0 }()",
     JavaCompiler -> "(someBool() ? 1 : 0)",
     JavaScriptCompiler -> "(this.someBool | 0)",
     LuaCompiler -> "self.some_bool and 1 or 0",
@@ -132,6 +133,7 @@ class TranslatorSpec extends FunSuite {
   full("foo_str", CalcStrType, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "foo_str()",
     CSharpCompiler -> "FooStr",
+    GoCompiler -> "this.FooStr",
     JavaCompiler -> "fooStr()",
     JavaScriptCompiler -> "this.fooStr",
     LuaCompiler -> "self.foo_str",
@@ -144,6 +146,7 @@ class TranslatorSpec extends FunSuite {
   full("foo_block", userType(List("block")), userType(List("block")), Map[LanguageCompilerStatic, String](
     CppCompiler -> "foo_block()",
     CSharpCompiler -> "FooBlock",
+    GoCompiler -> "this.FooBlock",
     JavaCompiler -> "fooBlock()",
     JavaScriptCompiler -> "this.fooBlock",
     LuaCompiler -> "self.foo_block",
@@ -156,6 +159,7 @@ class TranslatorSpec extends FunSuite {
   full("foo.bar", FooBarProvider, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "foo()->bar()",
     CSharpCompiler -> "Foo.Bar",
+    GoCompiler -> "this.foo.bar",
     JavaCompiler -> "foo().bar()",
     JavaScriptCompiler -> "this.foo.bar",
     LuaCompiler -> "self.foo.bar",
@@ -168,6 +172,7 @@ class TranslatorSpec extends FunSuite {
   full("foo.inner.baz", FooBarProvider, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "foo()->inner()->baz()",
     CSharpCompiler -> "Foo.Inner.Baz",
+    GoCompiler -> "Foo.Inner.Baz",
     JavaCompiler -> "foo().inner().baz()",
     JavaScriptCompiler -> "this.foo.inner.baz",
     LuaCompiler -> "self.foo.inner.baz",
@@ -180,6 +185,7 @@ class TranslatorSpec extends FunSuite {
   full("_root.foo", userType(List("top_class", "block")), userType(List("top_class", "block")), Map[LanguageCompilerStatic, String](
     CppCompiler -> "_root()->foo()",
     CSharpCompiler -> "M_Root.Foo",
+    GoCompiler -> "this._root.foo",
     JavaCompiler -> "_root.foo()",
     JavaScriptCompiler -> "this._root.foo",
     LuaCompiler -> "self._root.foo",
@@ -192,6 +198,7 @@ class TranslatorSpec extends FunSuite {
   full("a != 2 and a != 5", CalcIntType, CalcBooleanType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a() != 2 && a() != 5",
     CSharpCompiler -> "A != 2 && A != 5",
+    GoCompiler -> "a != 2 && a != 5",
     JavaCompiler -> "a() != 2 && a() != 5",
     JavaScriptCompiler -> "this.a != 2 && this.a != 5",
     LuaCompiler -> "self.a ~= 2 and self.a ~= 5",
@@ -204,6 +211,7 @@ class TranslatorSpec extends FunSuite {
   // Arrays
   full("[0, 1, 100500]", CalcIntType, ArrayType(CalcIntType), Map[LanguageCompilerStatic, String](
     CSharpCompiler -> "new List<int> { 0, 1, 100500 }",
+    GoCompiler -> "int[]{0, 1, 100500}",
     JavaCompiler -> "new ArrayList<Integer>(Arrays.asList(0, 1, 100500))",
     JavaScriptCompiler -> "[0, 1, 100500]",
     LuaCompiler -> "{0, 1, 100500}",
@@ -253,6 +261,7 @@ class TranslatorSpec extends FunSuite {
   full("a[42]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->at(42)",
     CSharpCompiler -> "A[42]",
+    GoCompiler -> "a[42]",
     JavaCompiler -> "a().get(42)",
     JavaScriptCompiler -> "this.a[42]",
     LuaCompiler -> "self.a[43]",
@@ -264,6 +273,7 @@ class TranslatorSpec extends FunSuite {
   full("a[42 - 2]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->at((42 - 2))",
     CSharpCompiler -> "A[(42 - 2)]",
+    GoCompiler -> "a[(42 - 2)]",
     JavaCompiler -> "a().get((42 - 2))",
     JavaScriptCompiler -> "this.a[(42 - 2)]",
     LuaCompiler -> "self.a[(43 - 2)]",
@@ -275,6 +285,7 @@ class TranslatorSpec extends FunSuite {
   full("a.first", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->front()",
     CSharpCompiler -> "A[0]",
+    GoCompiler -> "this.a[0]",
     JavaCompiler -> "a().get(0)",
     JavaScriptCompiler -> "this.a[0]",
     LuaCompiler -> "self.a[1]",
@@ -286,6 +297,7 @@ class TranslatorSpec extends FunSuite {
   full("a.last", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->back()",
     CSharpCompiler -> "A[A.Length - 1]",
+    GoCompiler -> "this.a[len(this.a)-1]",
     JavaCompiler -> "a().get(a().size() - 1)",
     JavaScriptCompiler -> "this.a[this.a.length - 1]",
     LuaCompiler -> "self.a[#self.a]",
@@ -297,6 +309,7 @@ class TranslatorSpec extends FunSuite {
   full("a.size", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->size()",
     CSharpCompiler -> "A.Count",
+    GoCompiler -> "len(this.a)",
     JavaCompiler -> "a().size()",
     JavaScriptCompiler -> "this.a.length",
     LuaCompiler -> "#self.a",
@@ -336,6 +349,7 @@ class TranslatorSpec extends FunSuite {
   full("\"str\\u000anext\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"str\\nnext\")",
     CSharpCompiler -> "\"str\\nnext\"",
+    GoCompiler -> "\"str\\u000anext\"",
     JavaCompiler -> "\"str\\nnext\"",
     JavaScriptCompiler -> "\"str\\nnext\"",
     LuaCompiler -> "\"str\\nnext\"",
@@ -348,6 +362,7 @@ class TranslatorSpec extends FunSuite {
   full("\"str\\0next\"", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"str\\000next\", 8)",
     CSharpCompiler -> "\"str\\0next\"",
+    GoCompiler -> "\"str\\000next\"",
     JavaCompiler -> "\"str\\000next\"",
     JavaScriptCompiler -> "\"str\\000next\"",
     LuaCompiler -> "\"str\\000next\"",
@@ -406,6 +421,7 @@ class TranslatorSpec extends FunSuite {
   full("\"str\".reverse", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
       CppCompiler -> "kaitai::kstream::reverse(std::string(\"str\"))",
       CSharpCompiler -> "new string(Array.Reverse(\"str\".ToCharArray()))",
+      GoCompiler -> "func(s string)(string)(r := []rune(s); for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 { r[i], r[j] = r[j], r[i] }; return string(r))(\"str\")",
       JavaCompiler -> "new StringBuilder(\"str\").reverse().toString()",
       JavaScriptCompiler -> "Array.from(\"str\").reverse().join('')",
       LuaCompiler -> "string.reverse(\"str\")",
@@ -418,6 +434,7 @@ class TranslatorSpec extends FunSuite {
   full("\"12345\".to_i", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::stoi(std::string(\"12345\"))",
     CSharpCompiler -> "Convert.ToInt64(\"12345\", 10)",
+    GoCompiler -> "func()(int){i, err := strconv.Atoi(\"12345\"); if (err != nil) { panic(err) }; return i}()",
     JavaCompiler -> "Long.parseLong(\"12345\", 10)",
     JavaScriptCompiler -> "Number.parseInt(\"12345\", 10)",
     LuaCompiler -> "tonumber(\"12345\")",
@@ -430,6 +447,7 @@ class TranslatorSpec extends FunSuite {
   full("\"1234fe\".to_i(16)", CalcIntType, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::stoi(std::string(\"1234fe\"), 0, 16)",
     CSharpCompiler -> "Convert.ToInt64(\"1234fe\", 16)",
+    GoCompiler -> "func()(int64){i, err := strconv.ParseInt(\"1234fe\", 16, 64); if (err != nil) { panic(err) }; return i}()",
     JavaCompiler -> "Long.parseLong(\"1234fe\", 16)",
     JavaScriptCompiler -> "Number.parseInt(\"1234fe\", 16)",
     LuaCompiler -> "tonumber(\"1234fe\", 16)",
@@ -443,6 +461,7 @@ class TranslatorSpec extends FunSuite {
   full("other.as<block>.bar", FooBarProvider, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "static_cast<block_t*>(other())->bar()",
     CSharpCompiler -> "((Block) (Other)).Bar",
+    GoCompiler -> "this.Other.(Block).Bar",
     JavaCompiler -> "((Block) (other())).bar()",
     JavaScriptCompiler -> "this.other.bar",
     LuaCompiler -> "self.other.bar",
@@ -455,6 +474,7 @@ class TranslatorSpec extends FunSuite {
   full("other.as<block::innerblock>.baz", FooBarProvider, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "static_cast<block_t::innerblock_t*>(other())->baz()",
     CSharpCompiler -> "((Block.Innerblock) (Other)).Baz",
+    GoCompiler -> "this.Other.(Block.Innerblock).Baz",
     JavaCompiler -> "((Block.Innerblock) (other())).baz()",
     JavaScriptCompiler -> "this.other.baz",
     LuaCompiler -> "self.other.baz",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 7a38c54b2..d710acb45 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -252,21 +252,20 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     }
   }
 
-//  override def strLength(s: Ast.expr): String =
-//    s"${translate(s)}.length()"
 //  override def strReverse(s: Ast.expr): String =
 //    s"new StringBuilder(${translate(s)}).reverse().toString()"
 //  override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String =
 //    s"${translate(s)}.substring(${translate(from)}, ${translate(to)})"
 
-//  override def arrayFirst(a: Ast.expr): String =
-//    s"${translate(a)}.get(0)"
-//  override def arrayLast(a: Ast.expr): String = {
-//    val v = translate(a)
-//    s"$v.get($v.size() - 1)"
-//  }
-//  override def arraySize(a: Ast.expr): String =
-//    s"${translate(a)}.size()"
+  override def arrayFirst(a: Ast.expr): TranslatorResult = {
+    ResultString(s"${translate(a)}[0]")
+  }
+  override def arrayLast(a: Ast.expr): TranslatorResult = {
+    ResultString(s"${translate(a)}[len(a)-1]")
+  }
+  override def arraySize(a: Ast.expr): TranslatorResult = {
+    ResultString(s"len(${translate(a)})")
+  }
 //  override def arrayMin(a: Ast.expr): String =
 //    s"Collections.min(${translate(a)})"
 //  override def arrayMax(a: Ast.expr): String =
@@ -315,12 +314,6 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
 
   override def kaitaiStreamPos(value: Ast.expr): TranslatorResult = ???
 
-  override def arrayFirst(a: Ast.expr): TranslatorResult = ???
-
-  override def arrayLast(a: Ast.expr): TranslatorResult = ???
-
-  override def arraySize(a: Ast.expr): TranslatorResult = ???
-
   override def arrayMin(a: Ast.expr): TranslatorResult = ???
 
   override def arrayMax(a: Ast.expr): TranslatorResult = ???

From 1a4ff869bb0d8f82dd914b612f0e04acfb9f113f Mon Sep 17 00:00:00 2001
From: Jonas Plum <cugu@users.noreply.github.com>
Date: Thu, 5 Apr 2018 08:20:28 +0200
Subject: [PATCH 109/230] Update TranslatorSpec.scala

---
 .../scala/io/kaitai/struct/translators/TranslatorSpec.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 9f8d09e2a..42458797b 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -159,7 +159,7 @@ class TranslatorSpec extends FunSuite {
   full("foo.bar", FooBarProvider, CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "foo()->bar()",
     CSharpCompiler -> "Foo.Bar",
-    GoCompiler -> "this.foo.bar",
+    GoCompiler -> "this.Foo.Bar",
     JavaCompiler -> "foo().bar()",
     JavaScriptCompiler -> "this.foo.bar",
     LuaCompiler -> "self.foo.bar",

From 1ec7567fd62779d0c9840903582a1efaddf8ea5f Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Thu, 5 Apr 2018 17:36:29 +0100
Subject: [PATCH 110/230] Successfully compiled hello world to Rust

---
 .../struct/languages/RustCompiler.scala       | 530 ++++++++++++++++++
 .../components/LanguageCompilerStatic.scala   |   3 +-
 .../struct/translators/RustTranslator.scala   | 135 +++++
 3 files changed, 667 insertions(+), 1 deletion(-)
 create mode 100644 shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
 create mode 100644 shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
new file mode 100644
index 000000000..f92ea92ff
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -0,0 +1,530 @@
+package io.kaitai.struct.languages
+
+import io.kaitai.struct._
+import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian}
+import io.kaitai.struct.exprlang.Ast
+import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _}
+import io.kaitai.struct.format._
+import io.kaitai.struct.languages.components._
+import io.kaitai.struct.translators.RustTranslator
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
+
+class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
+  extends LanguageCompiler(typeProvider, config)
+    with ObjectOrientedLanguage
+    with UpperCamelCaseClasses
+    with SingleOutputFile
+    with AllocateIOLocalVar
+    with UniversalFooter
+    with UniversalDoc
+    with FixedContentsUsingArrayByteLiteral
+    with EveryReadIsExpression {
+
+  import RustCompiler._
+
+  override def innerClasses = false
+
+  override def innerEnums = false
+
+  override val translator: RustTranslator = new RustTranslator(typeProvider, config)
+
+  val outStructDefns = new StringLanguageOutputWriter(indent)
+  val outPreInitialiser = new StringLanguageOutputWriter(indent)
+  val outInitialiser = new StringLanguageOutputWriter(indent)
+
+  override def results(topClass: ClassSpec): Map[String, String] = {
+    val fn = topClass.nameAsStr
+    Map(
+      s"$fn.rs" -> (outHeader.result + outStructDefns.result + outPreInitialiser.result + outInitialiser.result + out.result)
+    )
+  }
+
+  override def universalFooter: Unit = {
+    out.dec
+    out.puts("}")
+  }
+
+  override def indent: String = "    "
+  override def outFileName(topClassName: String): String = s"$topClassName.rs"
+
+  override def fileHeader(topClassName: String): Unit = {
+    outHeader.puts(s"// $headerComment")
+    outHeader.puts
+    
+    outHeader.puts("use std::{")
+    outHeader.puts("    option::Option,")
+    outHeader.puts("    boxed::Box,")
+    outHeader.puts("    io::Result")
+    outHeader.puts("};")
+    outHeader.puts
+
+    outHeader.puts("use kaitai_struct::{")
+    outHeader.puts("    KaitaiStream,")
+    outHeader.puts("    KaitaiStruct")
+    outHeader.puts("};")
+    outHeader.puts
+  }
+
+  override def classHeader(name: List[String]): Unit =
+    classHeader(name, Some(kstructName))
+
+  def classHeader(name: List[String], parentClass: Option[String]): Unit = {
+    outHeader.puts(s"pub struct ${type2class(name.last)} {")
+  }
+
+  override def classFooter(name: List[String]): Unit = universalFooter
+
+  override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = {
+    outPreInitialiser.puts("}")
+    outPreInitialiser.puts
+
+    outPreInitialiser.puts(s"impl KaitaiStruct for ${type2class(name.last)} {")
+    outPreInitialiser.inc
+    out.inc
+    
+    // Parameter names
+    val pIo = paramName(IoIdentifier)
+    val pParent = paramName(ParentIdentifier)
+    val pRoot = paramName(RootIdentifier)
+
+    // Types
+    val tIo = kstreamName
+    val tParent = kaitaiType2NativeType(parentType)
+
+    outPreInitialiser.puts(s"fn new<S: KaitaiStream>(stream: &mut S,")
+    outPreInitialiser.puts(s"                        _parent: &Option<Box<KaitaiStruct>>,")
+    outPreInitialiser.puts(s"                        _root: &Option<Box<KaitaiStruct>>)")
+    outPreInitialiser.puts(s"                        -> Result<Self>")
+    outPreInitialiser.inc
+    out.inc
+    outPreInitialiser.puts(s"where Self: Sized {")
+
+    outPreInitialiser.puts(s"let mut s = Self {")
+    
+    out.puts(s"};")
+    out.puts
+
+    out.puts(s"s.read(stream, _parent, _root)?;")
+    out.puts
+
+    out.puts("Ok(s)")
+  }
+
+  override def runRead(): Unit = {
+
+  }
+
+  override def runReadCalc(): Unit = {
+  
+  }
+
+  override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = {
+    out.puts
+    out.puts(s"fn read<S: KaitaiStream>(&mut self,")
+    out.puts(s"                         stream: &mut S,")
+    out.puts(s"                         _parent: &Option<Box<KaitaiStruct>>,")
+    out.puts(s"                         _root: &Option<Box<KaitaiStruct>>)")
+    out.puts(s"                         -> Result<()>")
+    out.inc
+    out.puts(s"where Self: Sized {")
+  }
+
+  override def readFooter(): Unit = {
+    out.puts
+    out.puts("Ok(())")
+    out.dec
+    out.puts("}")
+  }
+
+  override def attributeDeclaration(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {
+    attrName match {
+      case ParentIdentifier | RootIdentifier | IoIdentifier =>
+        // just ignore it for now
+      case _ =>
+        outStructDefns.puts(s"    pub ${idToStr(attrName)}: ${kaitaiType2NativeType(attrType)},")
+	outInitialiser.puts(s"            ${idToStr(attrName)}: ${kaitaiType2Default(attrType)},")
+    }
+  }
+
+  override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {
+  
+  }
+
+  override def universalDoc(doc: DocSpec): Unit = {
+    if (doc.summary.isDefined) {
+      out.puts
+      out.puts("/**")
+      doc.summary.foreach((summary) => out.putsLines(" * ", summary))
+      out.puts(" */")
+    }
+  }
+
+  override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = {
+    out.puts("if ($this->_m__is_le) {")
+    out.inc
+    leProc()
+    out.dec
+    out.puts("} else {")
+    out.inc
+    beProc()
+    out.dec
+    out.puts("}")
+  }
+
+  override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit =
+    out.puts(s"${privateMemberName(attrName)} = $normalIO->ensureFixedContents($contents);")
+
+  override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = {
+    val srcName = privateMemberName(varSrc)
+    val destName = privateMemberName(varDest)
+
+    proc match {
+      case ProcessXor(xorValue) =>
+        val procName = translator.detectType(xorValue) match {
+          case _: IntType => "processXorOne"
+          case _: BytesType => "processXorMany"
+        }
+        out.puts(s"$destName = $kstreamName::$procName($srcName, ${expression(xorValue)});")
+      case ProcessZlib =>
+        out.puts(s"$destName = $kstreamName::processZlib($srcName);")
+      case ProcessRotate(isLeft, rotValue) =>
+        val expr = if (isLeft) {
+          expression(rotValue)
+        } else {
+          s"8 - (${expression(rotValue)})"
+        }
+        out.puts(s"$destName = $kstreamName::processRotateLeft($srcName, $expr, 1);")
+      case ProcessCustom(name, args) =>
+        val isAbsolute = name.length > 1
+        val procClass = name.map((x) => type2class(x)).mkString(
+          if (isAbsolute) "\\" else "", "\\", ""
+        )
+        out.puts(s"$$_process = new $procClass(${args.map(expression).mkString(", ")});")
+        out.puts(s"$destName = $$_process->decode($srcName);")
+    }
+  }
+
+  override def allocateIO(id: Identifier, rep: RepeatSpec): String = {
+    val memberName = privateMemberName(id)
+
+    val args = rep match {
+      case RepeatEos | RepeatExpr(_) => s"end($memberName)"
+      case RepeatUntil(_) => translator.doLocalName(Identifier.ITERATOR2)
+      case NoRepeat => memberName
+    }
+
+    out.puts(s"$$io = new $kstreamName($args);")
+    "$io"
+  }
+
+  override def useIO(ioEx: Ast.expr): String = {
+    out.puts(s"$$io = ${expression(ioEx)};")
+    "$io"
+  }
+
+  override def pushPos(io: String): Unit =
+    out.puts(s"$$_pos = $io->pos();")
+
+  override def seek(io: String, pos: Ast.expr): Unit =
+    out.puts(s"$io->seek(${expression(pos)});")
+
+  override def popPos(io: String): Unit =
+    out.puts(s"$io->seek($$_pos);")
+
+  override def alignToByte(io: String): Unit =
+    out.puts(s"$io->alignToByte();")
+
+  override def condIfHeader(expr: Ast.expr): Unit = {
+    out.puts(s"if (${expression(expr)}) {")
+    out.inc
+  }
+
+  override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = {
+    if (needRaw)
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
+    out.puts(s"${privateMemberName(id)} = [];")
+    out.puts("$i = 0;")
+    out.puts(s"while (!$io->isEof()) {")
+    out.inc
+  }
+
+  override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = {
+    out.puts(s"${privateMemberName(id)}[] = $expr;")
+  }
+
+  override def condRepeatEosFooter: Unit = {
+    out.puts("$i++;")
+    super.condRepeatEosFooter
+  }
+
+  override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = {
+    if (needRaw)
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
+    out.puts(s"${privateMemberName(id)} = [];")
+    out.puts(s"$$n = ${expression(repeatExpr)};")
+    out.puts("for ($i = 0; $i < $n; $i++) {")
+    out.inc
+  }
+
+  override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = {
+    out.puts(s"${privateMemberName(id)}[] = $expr;")
+  }
+
+  override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
+    if (needRaw)
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
+    out.puts(s"${privateMemberName(id)} = [];")
+    out.puts("$i = 0;")
+    out.puts("do {")
+    out.inc
+  }
+
+  override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = {
+    val tmpName = translator.doLocalName(if (isRaw) Identifier.ITERATOR2 else Identifier.ITERATOR)
+    out.puts(s"$tmpName = $expr;")
+    out.puts(s"${privateMemberName(id)}[] = $tmpName;")
+  }
+
+  override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
+    typeProvider._currentIteratorType = Some(dataType)
+    out.puts("$i++;")
+    out.dec
+    out.puts(s"} while (!(${expression(untilExpr)}));")
+  }
+
+  override def handleAssignmentSimple(id: Identifier, expr: String): Unit = {
+    out.puts(s"${privateMemberName(id)} = $expr;")
+  }
+
+  override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = {
+    dataType match {
+      case t: ReadableType =>
+        s"$io.read_${t.apiCall(defEndian)}()?"
+      case blt: BytesLimitType =>
+        s"$io->readBytes(${expression(blt.size)})"
+      case _: BytesEosType =>
+        s"$io->readBytesFull()"
+      case BytesTerminatedType(terminator, include, consume, eosError, _) =>
+        s"$io->readBytesTerm($terminator, $include, $consume, $eosError)"
+      case BitsType1 =>
+        s"$io->readBitsInt(1) != 0"
+      case BitsType(width: Int) =>
+        s"$io->readBitsInt($width)"
+      case t: UserType =>
+        val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ")
+        val addArgs = if (t.isOpaque) {
+          ""
+        } else {
+          val parent = t.forcedParent match {
+            case Some(USER_TYPE_NO_PARENT) => "null"
+            case Some(fp) => translator.translate(fp)
+            case None => "$this"
+          }
+          val addEndian = t.classSpec.get.meta.endian match {
+            case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}"
+            case _ => ""
+          }
+          s", $parent, ${privateMemberName(RootIdentifier)}$addEndian"
+        }
+        s"new ${translator.types2classAbs(t.classSpec.get.name)}($addParams$io$addArgs)"
+    }
+  }
+
+  override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String = {
+    val expr1 = padRight match {
+      case Some(padByte) => s"$kstreamName::bytesStripRight($expr0, $padByte)"
+      case None => expr0
+    }
+    val expr2 = terminator match {
+      case Some(term) => s"$kstreamName::bytesTerminate($expr1, $term, $include)"
+      case None => expr1
+    }
+    expr2
+  }
+
+  override def switchStart(id: Identifier, on: Ast.expr): Unit = {
+    val onType = translator.detectType(on)
+
+    out.puts(s"switch (${expression(on)}) {")
+    out.inc
+  }
+
+  override def switchCaseStart(condition: Ast.expr): Unit = {
+    out.puts(s"case ${expression(condition)}:")
+    out.inc
+  }
+
+  override def switchCaseEnd(): Unit = {
+    out.puts("break;")
+    out.dec
+  }
+
+  override def switchElseStart(): Unit = {
+    out.puts("default:")
+    out.inc
+  }
+
+  override def switchEnd(): Unit = universalFooter
+
+  override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = {
+    out.puts(s"public function ${idToStr(instName)}() {")
+    out.inc
+  }
+
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+    out.puts(s"if (${privateMemberName(instName)} !== null)")
+    out.inc
+    instanceReturn(instName)
+    out.dec
+  }
+
+  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+    out.puts(s"return ${privateMemberName(instName)};")
+  }
+
+  override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = {
+    classHeader(curClass ::: List(enumName), None)
+    enumColl.foreach { case (id, label) =>
+      universalDoc(label.doc)
+      out.puts(s"const ${value2Const(label.name)} = $id;")
+    }
+    universalFooter
+  }
+
+  def value2Const(label: String) = label.toUpperCase
+
+  def idToStr(id: Identifier): String = {
+    id match {
+      case SpecialIdentifier(name) => name
+      case NamedIdentifier(name) => Utils.lowerCamelCase(name)
+      case NumberedIdentifier(idx) => s"_${NumberedIdentifier.TEMPLATE}$idx"
+      case InstanceIdentifier(name) => Utils.lowerCamelCase(name)
+      case RawIdentifier(innerId) => "_raw_" + idToStr(innerId)
+    }
+  }
+
+  override def privateMemberName(id: Identifier): String = {
+    id match {
+      case IoIdentifier => s"stream"
+      case RootIdentifier => s"_root"
+      case ParentIdentifier => s"_parent"
+      case _ => s"self.${idToStr(id)}"
+    }
+  }
+
+  override def publicMemberName(id: Identifier) = idToStr(id)
+
+  override def localTemporaryName(id: Identifier): String = s"$$_t_${idToStr(id)}"
+
+  override def paramName(id: Identifier): String = s"${idToStr(id)}"
+
+  /**
+    * Determine PHP data type corresponding to a KS data type. Currently unused due to
+    * problems with nullable types (which were introduced only in PHP 7.1).
+    *
+    * @param attrType KS data type
+    * @return PHP data type
+    */
+    
+  def kaitaiType2NativeType(attrType: DataType): String = {
+    attrType match {
+      case Int1Type(false) => "u8"
+      case IntMultiType(false, Width2, _) => "u16"
+      case IntMultiType(false, Width4, _) => "u32"
+      case IntMultiType(false, Width8, _) => "u64"
+
+      case Int1Type(true) => "i8"
+      case IntMultiType(true, Width2, _) => "i16"
+      case IntMultiType(true, Width4, _) => "i32"
+      case IntMultiType(true, Width8, _) => "i64"
+
+      case FloatMultiType(Width4, _) => "f32"
+      case FloatMultiType(Width8, _) => "f64"
+
+      case BitsType(_) => "u64"
+
+      case _: BooleanType => "bool"
+      case CalcIntType => "i32"
+      case CalcFloatType => "f64"
+
+      case _: StrType => "String"
+      case _: BytesType => "String"
+
+      case t: UserType => ""
+      case t: EnumType => ""
+      // TODO: figure this out
+
+      case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>*"
+
+      case KaitaiStreamType => s"$kstreamName"
+      case KaitaiStructType => s"$kstructName"
+      
+      case SwitchType(on, cases) => ""
+      // TODO
+    }
+  }
+  
+  def kaitaiType2Default(attrType: DataType): String = {
+    attrType match {
+      case Int1Type(false) => "0"
+      case IntMultiType(false, Width2, _) => "0"
+      case IntMultiType(false, Width4, _) => "0"
+      case IntMultiType(false, Width8, _) => "0"
+
+      case Int1Type(true) => "0"
+      case IntMultiType(true, Width2, _) => "0"
+      case IntMultiType(true, Width4, _) => "0"
+      case IntMultiType(true, Width8, _) => "0"
+
+      case FloatMultiType(Width4, _) => "f32"
+      case FloatMultiType(Width8, _) => "f64"
+
+      case BitsType(_) => "u64"
+
+      case _: BooleanType => "bool"
+      case CalcIntType => "i32"
+      case CalcFloatType => "f64"
+
+      case _: StrType => "String"
+      case _: BytesType => "String"
+
+      case t: UserType => ""
+      case t: EnumType => ""
+      // TODO: figure this out
+
+      case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>*"
+
+      case KaitaiStreamType => s"$kstreamName"
+      case KaitaiStructType => s"$kstructName"
+      
+      case SwitchType(on, cases) => ""
+      // TODO
+    }
+  }
+}
+
+object RustCompiler extends LanguageCompilerStatic
+  with StreamStructNames
+  with UpperCamelCaseClasses {
+  override def getCompiler(
+    tp: ClassTypeProvider,
+    config: RuntimeConfig
+  ): LanguageCompiler = new RustCompiler(tp, config)
+
+  override def kstructName = "&Option<Box<KaitaiStruct>>"
+  override def kstreamName = "&mut S"
+
+  def types2class(typeName: Ast.typeId) = {
+    typeName.names.map(type2class).mkString(
+      if (typeName.absolute) "::" else "",
+      "::",
+      ""
+    )
+  }
+
+  def types2classRel(names: List[String]) =
+    names.map(type2class).mkString("::")
+
+  override def type2class(name: String) = name
+}
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
index bc14394b9..3c8aacacc 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
@@ -21,7 +21,8 @@ object LanguageCompilerStatic {
     "perl" -> PerlCompiler,
     "php" -> PHPCompiler,
     "python" -> PythonCompiler,
-    "ruby" -> RubyCompiler
+    "ruby" -> RubyCompiler,
+    "rust" -> RustCompiler
   )
 
   val CLASS_TO_NAME: Map[LanguageCompilerStatic, String] = NAME_TO_CLASS.map(_.swap)
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
new file mode 100644
index 000000000..e8232ac77
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -0,0 +1,135 @@
+package io.kaitai.struct.translators
+
+import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.exprlang.Ast
+import io.kaitai.struct.exprlang.Ast.expr
+import io.kaitai.struct.format.Identifier
+import io.kaitai.struct.languages.RustCompiler
+import io.kaitai.struct.{RuntimeConfig, Utils}
+
+class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) {
+  override def doByteArrayLiteral(arr: Seq[Byte]): String =
+    "\"" + Utils.hexEscapeByteArray(arr) + "\""
+  override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
+    s"pack('C*', ${elts.map(translate).mkString(", ")})"
+
+  // http://php.net/manual/en/language.types.string.php#language.types.string.syntax.double
+  override val asciiCharQuoteMap: Map[Char, String] = Map(
+    '\t' -> "\\t",
+    '\n' -> "\\n",
+    '\r' -> "\\r",
+    '"' -> "\\\"",
+    '\\' -> "\\\\",
+
+    // allowed and required to not trigger variable interpolation
+    '$' -> "\\$",
+
+    '\f' -> "\\f",
+    '\13' -> "\\v",
+    '\33' -> "\\e"
+  )
+
+  override def strLiteralUnicode(code: Char): String =
+    "\\u{%x}".format(code.toInt)
+
+  override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = {
+    (detectType(left), detectType(right), op) match {
+      case (_: IntType, _: IntType, Ast.operator.Div) =>
+        s"intval(${translate(left)} / ${translate(right)})"
+      case (_: IntType, _: IntType, Ast.operator.Mod) =>
+        s"${RustCompiler.kstreamName}::mod(${translate(left)}, ${translate(right)})"
+      case _ =>
+        super.numericBinOp(left, op, right)
+    }
+  }
+
+  override def anyField(value: expr, attrName: String): String =
+    s"${translate(value)}->${doName(attrName)}"
+
+  override def doLocalName(s: String) = {
+    s match {
+      case Identifier.ITERATOR => "$_"
+      case Identifier.ITERATOR2 => "$_buf"
+      case Identifier.INDEX => "$i"
+      case _ => s"$$this->${doName(s)}"
+    }
+  }
+
+  override def doName(s: String) = s"${Utils.lowerCamelCase(s)}()"
+
+  override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = {
+    val enumClass = types2classAbs(enumTypeAbs)
+    s"$enumClass::${label.toUpperCase}"
+  }
+  override def doEnumById(enumTypeAbs: List[String], id: String) =
+    // Just an integer, without any casts / resolutions - one would have to look up constants manually
+    id
+
+  override def doSubscript(container: expr, idx: expr): String =
+    s"${translate(container)}[${translate(idx)}]"
+  override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
+    s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
+
+  // Predefined methods of various types
+  override def strConcat(left: Ast.expr, right: Ast.expr): String =
+    s"${translate(left)} . ${translate(right)}"
+
+  override def strToInt(s: expr, base: expr): String =
+    s"intval(${translate(s)}, ${translate(base)})"
+
+  override def enumToInt(v: expr, et: EnumType): String =
+    translate(v)
+
+  override def boolToInt(v: expr): String =
+    s"intval(${translate(v)})"
+
+  override def floatToInt(v: expr): String =
+    s"intval(${translate(v)})"
+
+  override def intToStr(i: expr, base: expr): String = {
+    val baseStr = translate(base)
+    baseStr match {
+      case "10" =>
+        s"strval(${translate(i)})"
+      case _ =>
+        s"base_convert(strval(${translate(i)}), 10, $baseStr)"
+    }
+  }
+  override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String =
+    s"${RustCompiler.kstreamName}::bytesToStr($bytesExpr, ${translate(encoding)})"
+  override def bytesLength(b: Ast.expr): String =
+    s"strlen(${translate(b)})"
+  override def strLength(s: expr): String =
+    s"strlen(${translate(s)})"
+  override def strReverse(s: expr): String =
+    s"strrev(${translate(s)})"
+  override def strSubstring(s: expr, from: expr, to: expr): String =
+    s"${translate(s)}.substring(${translate(from)}, ${translate(to)})"
+
+  override def arrayFirst(a: expr): String =
+    s"${translate(a)}[0]"
+  override def arrayLast(a: expr): String = {
+    // For huge debate on efficiency of PHP last element of array methods, see:
+    // http://stackoverflow.com/a/41795859/487064
+    val v = translate(a)
+    s"$v[count($v) - 1]"
+  }
+  override def arraySize(a: expr): String =
+    s"count(${translate(a)})"
+  override def arrayMin(a: Ast.expr): String =
+    s"min(${translate(a)})"
+  override def arrayMax(a: Ast.expr): String =
+    s"max(${translate(a)})"
+
+  val namespaceRef = if (config.phpNamespace.isEmpty) {
+    ""
+  } else {
+    "\\" + config.phpNamespace
+  }
+
+  def types2classAbs(names: List[String]) =
+    names match {
+      case List("kaitai_struct") => RustCompiler.kstructName
+      case _ => RustCompiler.types2classRel(names)
+    }
+}

From fa896a007e0e7f9ddcb0cc33911e212e2b004dcb Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 5 Apr 2018 22:26:34 +0100
Subject: [PATCH 111/230] GoTranslator: added subscript support, added boolean
 ops, fixed boolToInt and arrayLast to generate probably slightly more optimal
 non-inline versions

---
 .../struct/translators/GoTranslator.scala     | 30 +++++++++++++++----
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index d710acb45..0669b15c6 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -39,8 +39,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
         trStringLiteral(s)
       case Ast.expr.Bool(n) =>
         trBoolLiteral(n)
-
-//      case Ast.expr.BoolOp(op, values) =>
+      case Ast.expr.BoolOp(op, values) =>
+        trBooleanOp(op, values)
       case Ast.expr.BinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) =>
         (detectType(left), detectType(right), op) match {
           case (_: NumericType, _: NumericType, _) =>
@@ -72,7 +72,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
 //      case Ast.expr.EnumById(enumName, id) =>
       case ctt: Ast.expr.CastToType =>
         doCastOrArray(ctt)
-//      case Ast.expr.Subscript(value, idx) =>
+      case Ast.expr.Subscript(container, idx) =>
+        trSubscript(container, idx)
       case Ast.expr.Name(name: Ast.identifier) =>
         trLocalName(name.name)
       case Ast.expr.List(elts) =>
@@ -89,6 +90,9 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   def trStringLiteral(s: String): TranslatorResult = ResultString(doStringLiteral(s))
   def trBoolLiteral(n: Boolean): TranslatorResult = ResultString(doBoolLiteral(n))
 
+  def trBooleanOp(op: Ast.boolop, values: Seq[Ast.expr]) =
+    ResultString(doBooleanOp(op, values))
+
   def trNumericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) =
     ResultString(numericBinOp(left, op, right))
 
@@ -155,6 +159,9 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
       "_buf"
   }
 
+  def trSubscript(container: Ast.expr, idx: Ast.expr) =
+    ResultString(s"${translate(container)}[${translate(idx)}]")
+
   def trIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): ResultLocalVar = {
     val v1 = allocateLocalVar()
     val typ = detectType(ifTrue)
@@ -260,8 +267,10 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   override def arrayFirst(a: Ast.expr): TranslatorResult = {
     ResultString(s"${translate(a)}[0]")
   }
-  override def arrayLast(a: Ast.expr): TranslatorResult = {
-    ResultString(s"${translate(a)}[len(a)-1]")
+  override def arrayLast(a: Ast.expr): ResultString = {
+    val v = allocateLocalVar()
+    out.puts(s"${localVarName(v)} := ${translate(a)}")
+    ResultString(s"${localVarName(v)}[len(${localVarName(v)}) - 1]")
   }
   override def arraySize(a: Ast.expr): TranslatorResult = {
     ResultString(s"len(${translate(a)})")
@@ -320,7 +329,16 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
 
   override def enumToInt(value: Ast.expr, et: EnumType): TranslatorResult = ???
 
-  override def boolToInt(value: Ast.expr): TranslatorResult = ???
+  override def boolToInt(value: Ast.expr): ResultLocalVar = {
+    val v = allocateLocalVar()
+    out.puts(s"${localVarName(v)} := 0")
+    out.puts(s"if ${translate(value)} {")
+    out.inc
+    out.puts(s"${localVarName(v)} = 1")
+    out.dec
+    out.puts("}")
+    ResultLocalVar(v)
+  }
 
   def userType(dataType: UserType, io: String) = {
     val v = allocateLocalVar()

From 43fc8a539bd9d70a732e07bcc57a0c7c0f7a9de7 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 5 Apr 2018 22:27:02 +0100
Subject: [PATCH 112/230] TranslatorSpec: fixed some tests, mostly Go

---
 .../struct/translators/TranslatorSpec.scala   | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 42458797b..7d32e3fc5 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -119,7 +119,11 @@ class TranslatorSpec extends FunSuite {
   full("some_bool.to_i", CalcBooleanType, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "some_bool()",
     CSharpCompiler -> "(SomeBool ? 1 : 0)",
-    GoCompiler -> "func()(int) { if ( this.SomeBool ) { return 1 }; return 0 }()",
+    GoCompiler -> """tmp1 := 0
+                    |if this.SomeBool {
+                    |  tmp1 = 1
+                    |}
+                    |tmp1""".stripMargin,
     JavaCompiler -> "(someBool() ? 1 : 0)",
     JavaScriptCompiler -> "(this.someBool | 0)",
     LuaCompiler -> "self.some_bool and 1 or 0",
@@ -172,7 +176,7 @@ class TranslatorSpec extends FunSuite {
   full("foo.inner.baz", FooBarProvider, CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "foo()->inner()->baz()",
     CSharpCompiler -> "Foo.Inner.Baz",
-    GoCompiler -> "Foo.Inner.Baz",
+    GoCompiler -> "this.Foo.Inner.Baz",
     JavaCompiler -> "foo().inner().baz()",
     JavaScriptCompiler -> "this.foo.inner.baz",
     LuaCompiler -> "self.foo.inner.baz",
@@ -185,7 +189,7 @@ class TranslatorSpec extends FunSuite {
   full("_root.foo", userType(List("top_class", "block")), userType(List("top_class", "block")), Map[LanguageCompilerStatic, String](
     CppCompiler -> "_root()->foo()",
     CSharpCompiler -> "M_Root.Foo",
-    GoCompiler -> "this._root.foo",
+    GoCompiler -> "this._root.Foo",
     JavaCompiler -> "_root.foo()",
     JavaScriptCompiler -> "this._root.foo",
     LuaCompiler -> "self._root.foo",
@@ -211,7 +215,7 @@ class TranslatorSpec extends FunSuite {
   // Arrays
   full("[0, 1, 100500]", CalcIntType, ArrayType(CalcIntType), Map[LanguageCompilerStatic, String](
     CSharpCompiler -> "new List<int> { 0, 1, 100500 }",
-    GoCompiler -> "int[]{0, 1, 100500}",
+    GoCompiler -> "[]int{0, 1, 100500}",
     JavaCompiler -> "new ArrayList<Integer>(Arrays.asList(0, 1, 100500))",
     JavaScriptCompiler -> "[0, 1, 100500]",
     LuaCompiler -> "{0, 1, 100500}",
@@ -261,8 +265,8 @@ class TranslatorSpec extends FunSuite {
   full("a[42]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->at(42)",
     CSharpCompiler -> "A[42]",
-    GoCompiler -> "a[42]",
-    JavaCompiler -> "a().get(42)",
+    GoCompiler -> "this.A[42]",
+    JavaCompiler -> "a().get((int) 42)",
     JavaScriptCompiler -> "this.a[42]",
     LuaCompiler -> "self.a[43]",
     PHPCompiler -> "$this->a()[42]",
@@ -273,7 +277,7 @@ class TranslatorSpec extends FunSuite {
   full("a[42 - 2]", ArrayType(CalcStrType), CalcStrType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->at((42 - 2))",
     CSharpCompiler -> "A[(42 - 2)]",
-    GoCompiler -> "a[(42 - 2)]",
+    GoCompiler -> "this.A[(42 - 2)]",
     JavaCompiler -> "a().get((42 - 2))",
     JavaScriptCompiler -> "this.a[(42 - 2)]",
     LuaCompiler -> "self.a[(43 - 2)]",
@@ -285,7 +289,7 @@ class TranslatorSpec extends FunSuite {
   full("a.first", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->front()",
     CSharpCompiler -> "A[0]",
-    GoCompiler -> "this.a[0]",
+    GoCompiler -> "this.A[0]",
     JavaCompiler -> "a().get(0)",
     JavaScriptCompiler -> "this.a[0]",
     LuaCompiler -> "self.a[1]",
@@ -296,8 +300,8 @@ class TranslatorSpec extends FunSuite {
 
   full("a.last", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->back()",
-    CSharpCompiler -> "A[A.Length - 1]",
-    GoCompiler -> "this.a[len(this.a)-1]",
+    CSharpCompiler -> "A[A.Count - 1]",
+    GoCompiler -> "this.A[len(this.A)-1]",
     JavaCompiler -> "a().get(a().size() - 1)",
     JavaScriptCompiler -> "this.a[this.a.length - 1]",
     LuaCompiler -> "self.a[#self.a]",
@@ -309,7 +313,7 @@ class TranslatorSpec extends FunSuite {
   full("a.size", ArrayType(CalcIntType), CalcIntType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "a()->size()",
     CSharpCompiler -> "A.Count",
-    GoCompiler -> "len(this.a)",
+    GoCompiler -> "len(this.A)",
     JavaCompiler -> "a().size()",
     JavaScriptCompiler -> "this.a.length",
     LuaCompiler -> "#self.a",

From 89464db4808f34ede09b9df3332abeaf35f7e06d Mon Sep 17 00:00:00 2001
From: cugu <git@cugu.eu>
Date: Fri, 6 Apr 2018 09:18:17 +0200
Subject: [PATCH 113/230] Improve GoTranslator

---
 .../struct/translators/TranslatorSpec.scala   |  2 +-
 .../struct/translators/GoTranslator.scala     | 74 ++++++++++++++++---
 2 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 7d32e3fc5..87507ada1 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -425,7 +425,7 @@ class TranslatorSpec extends FunSuite {
   full("\"str\".reverse", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
       CppCompiler -> "kaitai::kstream::reverse(std::string(\"str\"))",
       CSharpCompiler -> "new string(Array.Reverse(\"str\".ToCharArray()))",
-      GoCompiler -> "func(s string)(string)(r := []rune(s); for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 { r[i], r[j] = r[j], r[i] }; return string(r))(\"str\")",
+      GoCompiler -> "func(s string)(string){r := []rune(s); for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 { r[i], r[j] = r[j], r[i] }; return string(r)}(\"str\")",
       JavaCompiler -> "new StringBuilder(\"str\").reverse().toString()",
       JavaScriptCompiler -> "Array.from(\"str\").reverse().join('')",
       LuaCompiler -> "string.reverse(\"str\")",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 0669b15c6..cace97a38 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -237,6 +237,9 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     "big5" -> ("traditionalchinese.Big5", "golang.org/x/text/encoding/traditionalchinese")
   )
 
+  override def bytesToStr(value: Ast.expr, expr: Ast.expr): TranslatorResult =
+    bytesToStr(translate(value), expr)
+
   def bytesToStr(bytesExpr: String, encoding: Ast.expr): TranslatorResult = {
     val enc = encoding match {
       case Ast.expr.Str(s) => s
@@ -304,28 +307,77 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     ResultString(s"utf8.RuneCountInString(${translate(s)})")
   }
 
-  override def strReverse(s: Ast.expr): TranslatorResult = ???
-
-  override def strToInt(s: Ast.expr, base: Ast.expr): TranslatorResult = ???
+  override def strReverse(s: Ast.expr): ResultLocalVar = {
+    val r = allocateLocalVar()
+    out.puts(s"${localVarName(r)} := []rune(${translate(s)})")
+    out.puts(s"for i, j := 0, len(${localVarName(r)})-1; i < len(${localVarName(r)})/2; i, j = i+1, j-1 {")
+    out.inc
+    out.puts(s"${localVarName(r)}[i], ${localVarName(r)}[j] = ${localVarName(r)}[j], ${localVarName(r)}[i]")
+    out.dec
+    out.puts("}")
+    ResultLocalVar(r)
+  }
 
-  override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): TranslatorResult = ???
+  override def strToInt(s: Ast.expr, base: Ast.expr): TranslatorResult = {
+    importList.add("strconv")
+    ResultString(s"strconv.ParseInt(${translate(s)}, ${translate(base)}, 0)")
+  }
 
-  override def bytesToStr(value: Ast.expr, expr: Ast.expr): TranslatorResult = ???
+  override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): TranslatorResult = {
+    ResultString(s"${translate(s)}[${translate(from)}:${translate(to)}]")
+  }
 
-  override def intToStr(value: Ast.expr, num: Ast.expr): TranslatorResult = ???
+  override def intToStr(value: Ast.expr, base: Ast.expr): TranslatorResult = {
+    importList.add("strconv")
+    ResultString(s"strconv.FormatInt(int64(${translate(value)}), ${translate(base)})")
+  }
 
   override def floatToInt(value: Ast.expr): TranslatorResult =
     ResultString(s"int(${translate(value)})")
 
-  override def kaitaiStreamSize(value: Ast.expr): TranslatorResult = ???
+  override def kaitaiStreamSize(value: Ast.expr): TranslatorResult = {
+    ResultString(s"${translate(value)}.Size()")
+  }
 
-  override def kaitaiStreamEof(value: Ast.expr): TranslatorResult = ???
+  override def kaitaiStreamEof(value: Ast.expr): TranslatorResult = {
+    ResultString(s"${translate(value)}.EOF()")
+  }
 
-  override def kaitaiStreamPos(value: Ast.expr): TranslatorResult = ???
+  override def kaitaiStreamPos(value: Ast.expr): TranslatorResult = {
+    ResultString(s"${translate(value)}.Pos()")
+  }
 
-  override def arrayMin(a: Ast.expr): TranslatorResult = ???
+  override def arrayMin(a: Ast.expr): ResultLocalVar = {
+    val min = allocateLocalVar()
+    val value = allocateLocalVar()
+    out.puts(s"${localVarName(min)} := ${translate(a)}[0]")
+    out.puts(s"for _, ${localVarName(value)} := range ${translate(a)} {")
+    out.inc
+    out.puts(s"if ${localVarName(min)} > ${localVarName(value)} {")
+    out.inc
+    out.puts(s"${localVarName(min)} = ${localVarName(value)}")
+    out.dec
+    out.puts("}")
+    out.dec
+    out.puts("}")
+    ResultLocalVar(min)
+  }
 
-  override def arrayMax(a: Ast.expr): TranslatorResult = ???
+  override def arrayMax(a: Ast.expr): ResultLocalVar = {
+    val max = allocateLocalVar()
+    val value = allocateLocalVar()
+    out.puts(s"${localVarName(max)} := ${translate(a)}[0]")
+    out.puts(s"for _, ${localVarName(value)} := range ${translate(a)} {")
+    out.inc
+    out.puts(s"if ${localVarName(max)} < ${localVarName(value)} {")
+    out.inc
+    out.puts(s"${localVarName(max)} = ${localVarName(value)}")
+    out.dec
+    out.puts("}")
+    out.dec
+    out.puts("}")
+    ResultLocalVar(max)
+  }
 
   override def enumToInt(value: Ast.expr, et: EnumType): TranslatorResult = ???
 

From 12a976b36c001d9492e2952d1f324fcd93b62ccf Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 7 Apr 2018 12:13:28 +0100
Subject: [PATCH 114/230] Fix root for ConstructTranslator

---
 .../io/kaitai/struct/translators/ConstructTranslator.scala      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
index 689c71aeb..1476d214c 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/ConstructTranslator.scala
@@ -9,7 +9,7 @@ class ConstructTranslator(provider: TypeProvider, importList: ImportList) extend
     s match {
       case Identifier.ITERATOR => "obj_"
       case Identifier.INDEX => "i"
-      case Identifier.ROOT => "_root"
+      case Identifier.ROOT => "this._root"
       case Identifier.IO => "_io"
       case _ => s"this.${doName(s)}"
     }

From 42e7bc845391d4d45f481e25c7978b4a0608697b Mon Sep 17 00:00:00 2001
From: cugu <git@cugu.eu>
Date: Sat, 7 Apr 2018 15:04:45 +0200
Subject: [PATCH 115/230] Use StringReverse and check output

---
 .../kaitai/struct/translators/TranslatorSpec.scala  |  2 +-
 .../io/kaitai/struct/translators/GoTranslator.scala | 13 +++----------
 2 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index 87507ada1..b1ca1db9e 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -425,7 +425,7 @@ class TranslatorSpec extends FunSuite {
   full("\"str\".reverse", CalcIntType, CalcStrType, Map[LanguageCompilerStatic, String](
       CppCompiler -> "kaitai::kstream::reverse(std::string(\"str\"))",
       CSharpCompiler -> "new string(Array.Reverse(\"str\".ToCharArray()))",
-      GoCompiler -> "func(s string)(string){r := []rune(s); for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 { r[i], r[j] = r[j], r[i] }; return string(r)}(\"str\")",
+      GoCompiler -> "kaitai.StringReverse(\"str\")",
       JavaCompiler -> "new StringBuilder(\"str\").reverse().toString()",
       JavaScriptCompiler -> "Array.from(\"str\").reverse().join('')",
       LuaCompiler -> "string.reverse(\"str\")",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index cace97a38..6288efd6a 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -308,14 +308,7 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   }
 
   override def strReverse(s: Ast.expr): ResultLocalVar = {
-    val r = allocateLocalVar()
-    out.puts(s"${localVarName(r)} := []rune(${translate(s)})")
-    out.puts(s"for i, j := 0, len(${localVarName(r)})-1; i < len(${localVarName(r)})/2; i, j = i+1, j-1 {")
-    out.inc
-    out.puts(s"${localVarName(r)}[i], ${localVarName(r)}[j] = ${localVarName(r)}[j], ${localVarName(r)}[i]")
-    out.dec
-    out.puts("}")
-    ResultLocalVar(r)
+    ResultString(s"kaitai.StringReverse(${translate(s)})")
   }
 
   override def strToInt(s: Ast.expr, base: Ast.expr): TranslatorResult = {
@@ -336,7 +329,7 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     ResultString(s"int(${translate(value)})")
 
   override def kaitaiStreamSize(value: Ast.expr): TranslatorResult = {
-    ResultString(s"${translate(value)}.Size()")
+    outVarCheckRes(s"${translate(value)}.Size()")
   }
 
   override def kaitaiStreamEof(value: Ast.expr): TranslatorResult = {
@@ -344,7 +337,7 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   }
 
   override def kaitaiStreamPos(value: Ast.expr): TranslatorResult = {
-    ResultString(s"${translate(value)}.Pos()")
+    outVarCheckRes(s"${translate(value)}.Pos()")
   }
 
   override def arrayMin(a: Ast.expr): ResultLocalVar = {

From 34f006fc21fb17b164dcdae99d76375f60f1641e Mon Sep 17 00:00:00 2001
From: Jonas Plum <cugu@users.noreply.github.com>
Date: Sun, 8 Apr 2018 08:48:29 +0200
Subject: [PATCH 116/230] Fix StringReverse

---
 .../main/scala/io/kaitai/struct/translators/GoTranslator.scala  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 6288efd6a..b9229024b 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -307,7 +307,7 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     ResultString(s"utf8.RuneCountInString(${translate(s)})")
   }
 
-  override def strReverse(s: Ast.expr): ResultLocalVar = {
+  override def strReverse(s: Ast.expr): TranslatorResult = {
     ResultString(s"kaitai.StringReverse(${translate(s)})")
   }
 

From 4128d9f25428f16e5d592f23deb08f0d567fa17e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 8 Apr 2018 13:15:33 +0100
Subject: [PATCH 117/230] GoTranslator: general cleanup, make byte array
 literals generate uint8 arrays

---
 .../struct/translators/GoTranslator.scala     | 34 +++----------------
 1 file changed, 5 insertions(+), 29 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index b9229024b..29fe050c8 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -3,7 +3,7 @@ package io.kaitai.struct.translators
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
-import io.kaitai.struct.format.{ClassSpec, Identifier}
+import io.kaitai.struct.format.Identifier
 import io.kaitai.struct.languages.GoCompiler
 import io.kaitai.struct.precompile.TypeMismatchError
 import io.kaitai.struct.{ImportList, StringLanguageOutputWriter, Utils}
@@ -105,15 +105,6 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   def trStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult =
     ResultString(doStrCompareOp(left, op, right))
 
-//  override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = {
-//    val javaType = JavaCompiler.kaitaiType2JavaTypeBoxed(t)
-//    val commaStr = value.map((v) => translate(v)).mkString(", ")
-//    s"new ArrayList<$javaType>(Arrays.asList($commaStr))"
-//  }
-//
-//  override def doByteArrayLiteral(arr: Seq[Byte]): String =
-//    s"new byte[] { ${arr.mkString(", ")} }"
-
   override def unaryOp(op: Ast.unaryop): String = op match {
     case Ast.unaryop.Invert => "^"
     case Ast.unaryop.Minus => "-"
@@ -199,31 +190,18 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     }
   }
 
-//  override def doSubscript(container: Ast.expr, idx: Ast.expr): String =
-//    s"${translate(container)}.get((int) ${translate(idx)})"
-//  override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String =
-//    s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
-//  override def doCast(value: Ast.expr, typeName: String): String =
-//    s"((${Utils.upperCamelCase(typeName)}) (${translate(value)}))"
-
   override def doCast(value: Ast.expr, typeName: Ast.typeId): TranslatorResult = ???
 
   override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]) =
     ResultString(s"[]${GoCompiler.kaitaiType2NativeType(t)}{${value.map(translate).mkString(", ")}}")
 
   override def doByteArrayLiteral(arr: Seq[Byte]): TranslatorResult =
-    ResultString("\"" + Utils.hexEscapeByteArray(arr) + "\"")
+    ResultString("[]uint8{" + arr.map(_ & 0xff).mkString(", ") + "}")
 
   override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): TranslatorResult =
-    ResultString("string([]byte{" + elts.map(translate).mkString(", ") + "})")
+    ResultString("[]uint8{" + elts.map(translate).mkString(", ") + "}")
 
   // Predefined methods of various types
-//  override def strToInt(s: Ast.expr, base: Ast.expr): String =
-//    s"Long.parseLong(${translate(s)}, ${translate(base)})"
-//  override def enumToInt(v: Ast.expr, et: EnumType): String =
-//    s"${translate(v)}.id()"
-//  override def intToStr(i: Ast.expr, base: Ast.expr): String =
-//    s"Long.toString(${translate(i)}, ${translate(base)})"
 
   val IMPORT_CHARMAP = "golang.org/x/text/encoding/charmap"
 
@@ -267,17 +245,15 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
 //  override def strSubstring(s: Ast.expr, from: Ast.expr, to: Ast.expr): String =
 //    s"${translate(s)}.substring(${translate(from)}, ${translate(to)})"
 
-  override def arrayFirst(a: Ast.expr): TranslatorResult = {
+  override def arrayFirst(a: Ast.expr): TranslatorResult =
     ResultString(s"${translate(a)}[0]")
-  }
   override def arrayLast(a: Ast.expr): ResultString = {
     val v = allocateLocalVar()
     out.puts(s"${localVarName(v)} := ${translate(a)}")
     ResultString(s"${localVarName(v)}[len(${localVarName(v)}) - 1]")
   }
-  override def arraySize(a: Ast.expr): TranslatorResult = {
+  override def arraySize(a: Ast.expr): TranslatorResult =
     ResultString(s"len(${translate(a)})")
-  }
 //  override def arrayMin(a: Ast.expr): String =
 //    s"Collections.min(${translate(a)})"
 //  override def arrayMax(a: Ast.expr): String =

From 783d4fc938b03a636134a7e11396db7ba5bdd9c0 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 17:54:45 +0100
Subject: [PATCH 118/230] Produced significant part of Rust output compilation

---
 .../main/scala/io/kaitai/struct/Main.scala    |   4 +-
 .../io/kaitai/struct/RustClassCompiler.scala  | 104 +++++++++
 .../struct/languages/RustCompiler.scala       | 199 +++++++++---------
 .../struct/translators/RustTranslator.scala   |  44 ++--
 4 files changed, 218 insertions(+), 133 deletions(-)
 create mode 100644 shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala

diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala
index d3f1374de..299185072 100644
--- a/shared/src/main/scala/io/kaitai/struct/Main.scala
+++ b/shared/src/main/scala/io/kaitai/struct/Main.scala
@@ -1,7 +1,7 @@
 package io.kaitai.struct
 
 import io.kaitai.struct.format.{ClassSpec, ClassSpecs, GenericStructClassSpec}
-import io.kaitai.struct.languages.GoCompiler
+import io.kaitai.struct.languages.{GoCompiler, RustCompiler}
 import io.kaitai.struct.languages.components.LanguageCompilerStatic
 import io.kaitai.struct.precompile._
 
@@ -59,6 +59,8 @@ object Main {
         new GraphvizClassCompiler(specs, spec)
       case GoCompiler =>
         new GoClassCompiler(specs, spec, config)
+      case RustCompiler =>
+        new RustClassCompiler(specs, spec, config)
       case ConstructClassCompiler =>
         new ConstructClassCompiler(specs, spec)
       case _ =>
diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
new file mode 100644
index 000000000..36dd17864
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
@@ -0,0 +1,104 @@
+package io.kaitai.struct
+
+import io.kaitai.struct.datatype.DataType.{KaitaiStreamType, UserTypeInstream}
+import io.kaitai.struct.datatype.{Endianness, FixedEndian, InheritedEndian}
+import io.kaitai.struct.format._
+import io.kaitai.struct.languages.RustCompiler
+import io.kaitai.struct.languages.components.ExtraAttrs
+
+import scala.collection.mutable.ListBuffer
+
+class RustClassCompiler(
+  classSpecs: ClassSpecs,
+  override val topClass: ClassSpec,
+  config: RuntimeConfig
+) extends ClassCompiler(classSpecs, topClass, config, RustCompiler) {
+
+  override def compileClass(curClass: ClassSpec): Unit = {
+    provider.nowClass = curClass
+
+    val extraAttrs = ListBuffer[AttrSpec]()
+    extraAttrs += AttrSpec(List(), IoIdentifier, KaitaiStreamType)
+    extraAttrs += AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None))
+    extraAttrs += AttrSpec(List(), ParentIdentifier, curClass.parentType)
+
+    extraAttrs ++= getExtraAttrs(curClass)
+
+    if (!curClass.doc.isEmpty)
+      lang.classDoc(curClass.name, curClass.doc)
+
+    // Basic struct declaration
+    lang.classHeader(curClass.name)
+    
+    compileAttrDeclarations(curClass.seq ++ extraAttrs)
+    curClass.instances.foreach { case (instName, instSpec) =>
+      compileInstanceDeclaration(instName, instSpec)
+    }
+    
+    // Constructor = Read() function
+    compileReadFunction(curClass, extraAttrs)
+    
+
+    compileInstances(curClass, extraAttrs)
+
+    compileAttrReaders(curClass.seq ++ extraAttrs)
+    lang.classFooter(curClass.name)
+
+    compileEnums(curClass)
+
+    // Recursive types
+    compileSubclasses(curClass)
+  }
+
+  def compileReadFunction(curClass: ClassSpec, extraAttrs: ListBuffer[AttrSpec]) = {
+    lang.classConstructorHeader(
+      curClass.name,
+      curClass.parentType,
+      topClassName,
+      curClass.meta.endian.contains(InheritedEndian),
+      curClass.params
+    )
+
+    // FIXME
+    val defEndian = curClass.meta.endian match {
+      case Some(fe: FixedEndian) => Some(fe)
+      case _ => None
+    }
+    
+    lang.readHeader(defEndian, false)
+    
+    compileSeq(curClass.seq, extraAttrs, defEndian)
+    lang.classConstructorFooter
+  }
+
+  override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, extraAttrs: ListBuffer[AttrSpec], endian: Option[Endianness]): Unit = {
+    // FIXME: support calculated endianness
+
+    // Determine datatype
+    val dataType = instSpec.dataTypeComposite
+
+    if (!instSpec.doc.isEmpty)
+      lang.attributeDoc(instName, instSpec.doc)
+    lang.instanceHeader(className, instName, dataType, instSpec.isNullable)
+    lang.instanceCheckCacheAndReturn(instName)
+
+    instSpec match {
+      case vi: ValueInstanceSpec =>
+        lang.attrParseIfHeader(instName, vi.ifExpr)
+        lang.instanceCalculate(instName, dataType, vi.value)
+        lang.attrParseIfFooter(vi.ifExpr)
+      case i: ParseInstanceSpec =>
+        lang.attrParse(i, instName, extraAttrs, None) // FIXME
+    }
+
+    lang.instanceSetCalculated(instName)
+    lang.instanceReturn(instName)
+    lang.instanceFooter
+  }
+
+  def getExtraAttrs(curClass: ClassSpec): List[AttrSpec] = {
+    curClass.seq.foldLeft(List[AttrSpec]())(
+      (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr)
+    )
+  }
+}
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index f92ea92ff..ce2fdd2ce 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -7,7 +7,7 @@ import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _}
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
-import io.kaitai.struct.translators.RustTranslator
+import io.kaitai.struct.translators.{RustTranslator, TypeDetector}
 import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
 
 class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
@@ -29,14 +29,10 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override val translator: RustTranslator = new RustTranslator(typeProvider, config)
 
-  val outStructDefns = new StringLanguageOutputWriter(indent)
-  val outPreInitialiser = new StringLanguageOutputWriter(indent)
-  val outInitialiser = new StringLanguageOutputWriter(indent)
-
   override def results(topClass: ClassSpec): Map[String, String] = {
     val fn = topClass.nameAsStr
     Map(
-      s"$fn.rs" -> (outHeader.result + outStructDefns.result + outPreInitialiser.result + outInitialiser.result + out.result)
+      s"$fn.rs" -> out.result
     )
   }
 
@@ -49,38 +45,39 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def outFileName(topClassName: String): String = s"$topClassName.rs"
 
   override def fileHeader(topClassName: String): Unit = {
-    outHeader.puts(s"// $headerComment")
-    outHeader.puts
+    out.puts(s"// $headerComment")
+    out.puts
     
-    outHeader.puts("use std::{")
-    outHeader.puts("    option::Option,")
-    outHeader.puts("    boxed::Box,")
-    outHeader.puts("    io::Result")
-    outHeader.puts("};")
-    outHeader.puts
+    out.puts("use std::{")
+    out.puts("    option::Option,")
+    out.puts("    boxed::Box,")
+    out.puts("    io::{Result, Cursor},")
+    out.puts("    default::Default")
+    out.puts("};")
+    out.puts
 
-    outHeader.puts("use kaitai_struct::{")
-    outHeader.puts("    KaitaiStream,")
-    outHeader.puts("    KaitaiStruct")
-    outHeader.puts("};")
-    outHeader.puts
+    out.puts("use kaitai_struct::{")
+    out.puts("    KaitaiStream,")
+    out.puts("    KaitaiStruct")
+    out.puts("};")
+    out.puts
   }
 
   override def classHeader(name: List[String]): Unit =
     classHeader(name, Some(kstructName))
 
   def classHeader(name: List[String], parentClass: Option[String]): Unit = {
-    outHeader.puts(s"pub struct ${type2class(name.last)} {")
+    out.puts("#[derive(Default)]")
+    out.puts(s"pub struct ${type2class(name)} {")
   }
 
   override def classFooter(name: List[String]): Unit = universalFooter
 
   override def classConstructorHeader(name: List[String], parentType: DataType, rootClassName: List[String], isHybrid: Boolean, params: List[ParamDefSpec]): Unit = {
-    outPreInitialiser.puts("}")
-    outPreInitialiser.puts
+    out.puts("}")
+    out.puts
 
-    outPreInitialiser.puts(s"impl KaitaiStruct for ${type2class(name.last)} {")
-    outPreInitialiser.inc
+    out.puts(s"impl KaitaiStruct for ${type2class(name)} {")
     out.inc
     
     // Parameter names
@@ -92,23 +89,23 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     val tIo = kstreamName
     val tParent = kaitaiType2NativeType(parentType)
 
-    outPreInitialiser.puts(s"fn new<S: KaitaiStream>(stream: &mut S,")
-    outPreInitialiser.puts(s"                        _parent: &Option<Box<KaitaiStruct>>,")
-    outPreInitialiser.puts(s"                        _root: &Option<Box<KaitaiStruct>>)")
-    outPreInitialiser.puts(s"                        -> Result<Self>")
-    outPreInitialiser.inc
+    out.puts(s"fn new<S: KaitaiStream>(stream: &mut S,")
+    out.puts(s"                        _parent: &Option<Box<KaitaiStruct>>,")
+    out.puts(s"                        _root: &Option<Box<KaitaiStruct>>)")
+    out.puts(s"                        -> Result<Self>")
     out.inc
-    outPreInitialiser.puts(s"where Self: Sized {")
+    out.puts(s"where Self: Sized {")
 
-    outPreInitialiser.puts(s"let mut s = Self {")
-    
-    out.puts(s"};")
+    out.puts(s"let mut s: Self = Default::default();")
     out.puts
 
     out.puts(s"s.read(stream, _parent, _root)?;")
     out.puts
 
     out.puts("Ok(s)")
+    out.dec
+    out.puts("}")
+    out.puts
   }
 
   override def runRead(): Unit = {
@@ -142,8 +139,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case ParentIdentifier | RootIdentifier | IoIdentifier =>
         // just ignore it for now
       case _ =>
-        outStructDefns.puts(s"    pub ${idToStr(attrName)}: ${kaitaiType2NativeType(attrType)},")
-	outInitialiser.puts(s"            ${idToStr(attrName)}: ${kaitaiType2Default(attrType)},")
+        out.puts(s"    pub ${idToStr(attrName)}: ${kaitaiType2NativeType(attrType)},")
     }
   }
 
@@ -154,7 +150,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def universalDoc(doc: DocSpec): Unit = {
     if (doc.summary.isDefined) {
       out.puts
-      out.puts("/**")
+      out.puts("/*")
       doc.summary.foreach((summary) => out.putsLines(" * ", summary))
       out.puts(" */")
     }
@@ -200,8 +196,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
         val procClass = name.map((x) => type2class(x)).mkString(
           if (isAbsolute) "\\" else "", "\\", ""
         )
-        out.puts(s"$$_process = new $procClass(${args.map(expression).mkString(", ")});")
-        out.puts(s"$destName = $$_process->decode($srcName);")
+        out.puts(s"let _process = $procClass::new(${args.map(expression).mkString(", ")});")
+        out.puts(s"$destName = _process.decode($srcName);")
     }
   }
 
@@ -214,29 +210,29 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case NoRepeat => memberName
     }
 
-    out.puts(s"$$io = new $kstreamName($args);")
-    "$io"
+    out.puts(s"let mut io = Cursor::new($args);")
+    "io"
   }
 
   override def useIO(ioEx: Ast.expr): String = {
-    out.puts(s"$$io = ${expression(ioEx)};")
-    "$io"
+    out.puts(s"io = ${expression(ioEx)};")
+    "io"
   }
 
   override def pushPos(io: String): Unit =
-    out.puts(s"$$_pos = $io->pos();")
+    out.puts(s"let _pos = $io.pos();")
 
   override def seek(io: String, pos: Ast.expr): Unit =
-    out.puts(s"$io->seek(${expression(pos)});")
+    out.puts(s"$io.seek(${expression(pos)});")
 
   override def popPos(io: String): Unit =
-    out.puts(s"$io->seek($$_pos);")
+    out.puts(s"$io.seek(_pos);")
 
   override def alignToByte(io: String): Unit =
-    out.puts(s"$io->alignToByte();")
+    out.puts(s"$io.alignToByte();")
 
   override def condIfHeader(expr: Ast.expr): Unit = {
-    out.puts(s"if (${expression(expr)}) {")
+    out.puts(s"if ${expression(expr)} {")
     out.inc
   }
 
@@ -244,31 +240,28 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     if (needRaw)
       out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
     out.puts(s"${privateMemberName(id)} = [];")
-    out.puts("$i = 0;")
-    out.puts(s"while (!$io->isEof()) {")
+    out.puts(s"while !$io.isEof() {")
     out.inc
   }
 
   override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = {
-    out.puts(s"${privateMemberName(id)}[] = $expr;")
+    out.puts(s"${privateMemberName(id)}.push($expr);")
   }
 
   override def condRepeatEosFooter: Unit = {
-    out.puts("$i++;")
     super.condRepeatEosFooter
   }
 
   override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
-    out.puts(s"${privateMemberName(id)} = [];")
-    out.puts(s"$$n = ${expression(repeatExpr)};")
-    out.puts("for ($i = 0; $i < $n; $i++) {")
+      out.puts(s"${privateMemberName(RawIdentifier(id))}: Vector<${kaitaiType2NativeType(dataType)}> = vec!();")
+    out.puts(s"${privateMemberName(id)}: Vector<${kaitaiType2NativeType(dataType)}> = vec!();")
+    out.puts(s"for i in 0..${expression(repeatExpr)} {")
     out.inc
   }
 
   override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = {
-    out.puts(s"${privateMemberName(id)}[] = $expr;")
+    out.puts(s"${privateMemberName(id)}.push($expr);")
   }
 
   override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
@@ -302,15 +295,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case t: ReadableType =>
         s"$io.read_${t.apiCall(defEndian)}()?"
       case blt: BytesLimitType =>
-        s"$io->readBytes(${expression(blt.size)})"
+        s"$io.read_bytes(${expression(blt.size)})"
       case _: BytesEosType =>
-        s"$io->readBytesFull()"
+        s"$io.read_bytes_full()"
       case BytesTerminatedType(terminator, include, consume, eosError, _) =>
-        s"$io->readBytesTerm($terminator, $include, $consume, $eosError)"
+        s"$io.read_bytes_term($terminator, $include, $consume, $eosError)"
       case BitsType1 =>
-        s"$io->readBitsInt(1) != 0"
+        s"$io.read_bits_int(1) != 0"
       case BitsType(width: Int) =>
-        s"$io->readBitsInt($width)"
+        s"$io.read_bits_int($width)"
       case t: UserType =>
         val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ")
         val addArgs = if (t.isOpaque) {
@@ -319,7 +312,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           val parent = t.forcedParent match {
             case Some(USER_TYPE_NO_PARENT) => "null"
             case Some(fp) => translator.translate(fp)
-            case None => "$this"
+            case None => "self"
           }
           val addEndian = t.classSpec.get.meta.endian match {
             case Some(InheritedEndian) => s", ${privateMemberName(EndianIdentifier)}"
@@ -327,7 +320,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           }
           s", $parent, ${privateMemberName(RootIdentifier)}$addEndian"
         }
-        s"new ${translator.types2classAbs(t.classSpec.get.name)}($addParams$io$addArgs)"
+        s"${translator.types2classAbs(t.classSpec.get.name)}::new(stream, self, _root)"
     }
   }
 
@@ -346,37 +339,43 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def switchStart(id: Identifier, on: Ast.expr): Unit = {
     val onType = translator.detectType(on)
 
-    out.puts(s"switch (${expression(on)}) {")
+    out.puts(s"match ${expression(on)} {")
     out.inc
   }
 
   override def switchCaseStart(condition: Ast.expr): Unit = {
-    out.puts(s"case ${expression(condition)}:")
+    out.puts(s"${expression(condition)} => {")
     out.inc
   }
 
   override def switchCaseEnd(): Unit = {
-    out.puts("break;")
     out.dec
+    out.puts("},")
   }
 
   override def switchElseStart(): Unit = {
-    out.puts("default:")
+    out.puts("_ => {")
     out.inc
   }
 
   override def switchEnd(): Unit = universalFooter
 
+  override def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = {
+    out.puts(s"    pub ${idToStr(attrName)}: Option<${kaitaiType2NativeType(attrType)}>,")
+  }
+
   override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = {
-    out.puts(s"public function ${idToStr(instName)}() {")
+    out.puts(s"fn ${idToStr(instName)}() -> ${kaitaiType2NativeType(dataType)} {")
     out.inc
   }
 
   override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
-    out.puts(s"if (${privateMemberName(instName)} !== null)")
+    out.puts(s"if let Some(x) = ${privateMemberName(instName)} {")
     out.inc
-    instanceReturn(instName)
+    out.puts("return x;")
     out.dec
+    out.puts("}")
+    out.puts
   }
 
   override def instanceReturn(instName: InstanceIdentifier): Unit = {
@@ -384,12 +383,18 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = {
-    classHeader(curClass ::: List(enumName), None)
+    val enumClass = type2class(curClass ::: List(enumName))
+
+    out.puts(s"enum $enumClass {")
+    out.inc
+    
     enumColl.foreach { case (id, label) =>
       universalDoc(label.doc)
-      out.puts(s"const ${value2Const(label.name)} = $id;")
+      out.puts(s"${value2Const(label.name)},")
     }
-    universalFooter
+
+    out.dec
+    out.puts("}")
   }
 
   def value2Const(label: String) = label.toUpperCase
@@ -418,14 +423,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def localTemporaryName(id: Identifier): String = s"$$_t_${idToStr(id)}"
 
   override def paramName(id: Identifier): String = s"${idToStr(id)}"
-
-  /**
-    * Determine PHP data type corresponding to a KS data type. Currently unused due to
-    * problems with nullable types (which were introduced only in PHP 7.1).
-    *
-    * @param attrType KS data type
-    * @return PHP data type
-    */
     
   def kaitaiType2NativeType(attrType: DataType): String = {
     attrType match {
@@ -449,19 +446,17 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case CalcFloatType => "f64"
 
       case _: StrType => "String"
-      case _: BytesType => "String"
+      case _: BytesType => "Vec<u8>"
 
-      case t: UserType => ""
-      case t: EnumType => ""
-      // TODO: figure this out
+      case t: UserType => types2classRel(t.name)
+      case t: EnumType => types2classRel(t.name)
 
-      case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>*"
+      case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>"
 
       case KaitaiStreamType => s"$kstreamName"
       case KaitaiStructType => s"$kstructName"
       
-      case SwitchType(on, cases) => ""
-      // TODO
+      case SwitchType(on, cases) => kaitaiType2NativeType(TypeDetector.combineTypes(cases.values))
     }
   }
   
@@ -477,23 +472,23 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case IntMultiType(true, Width4, _) => "0"
       case IntMultiType(true, Width8, _) => "0"
 
-      case FloatMultiType(Width4, _) => "f32"
-      case FloatMultiType(Width8, _) => "f64"
+      case FloatMultiType(Width4, _) => "0"
+      case FloatMultiType(Width8, _) => "0"
 
-      case BitsType(_) => "u64"
+      case BitsType(_) => "0"
 
-      case _: BooleanType => "bool"
-      case CalcIntType => "i32"
-      case CalcFloatType => "f64"
+      case _: BooleanType => "false"
+      case CalcIntType => "0"
+      case CalcFloatType => "0"
 
-      case _: StrType => "String"
-      case _: BytesType => "String"
+      case _: StrType => "\"\""
+      case _: BytesType => "vec!()"
 
       case t: UserType => ""
       case t: EnumType => ""
       // TODO: figure this out
 
-      case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>*"
+      case ArrayType(inType) => s"vec!()"
 
       case KaitaiStreamType => s"$kstreamName"
       case KaitaiStructType => s"$kstructName"
@@ -502,6 +497,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       // TODO
     }
   }
+  
+  def type2class(names: List[String]) = types2classRel(names)
 }
 
 object RustCompiler extends LanguageCompilerStatic
@@ -517,14 +514,12 @@ object RustCompiler extends LanguageCompilerStatic
 
   def types2class(typeName: Ast.typeId) = {
     typeName.names.map(type2class).mkString(
-      if (typeName.absolute) "::" else "",
-      "::",
+      if (typeName.absolute) "__" else "",
+      "__",
       ""
     )
   }
 
   def types2classRel(names: List[String]) =
-    names.map(type2class).mkString("::")
-
-  override def type2class(name: String) = name
+    names.map(type2class).mkString("__")
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index e8232ac77..46fed1e4c 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -9,11 +9,12 @@ import io.kaitai.struct.{RuntimeConfig, Utils}
 
 class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) {
   override def doByteArrayLiteral(arr: Seq[Byte]): String =
-    "\"" + Utils.hexEscapeByteArray(arr) + "\""
+    "vec!([" + arr.map((x) =>
+    	     "0x%0.2X".format(x & 0xff)
+    ).mkString(", ") + "])"
   override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
     s"pack('C*', ${elts.map(translate).mkString(", ")})"
 
-  // http://php.net/manual/en/language.types.string.php#language.types.string.syntax.double
   override val asciiCharQuoteMap: Map[Char, String] = Map(
     '\t' -> "\\t",
     '\n' -> "\\n",
@@ -21,9 +22,6 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
     '"' -> "\\\"",
     '\\' -> "\\\\",
 
-    // allowed and required to not trigger variable interpolation
-    '$' -> "\\$",
-
     '\f' -> "\\f",
     '\13' -> "\\v",
     '\33' -> "\\e"
@@ -43,19 +41,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
     }
   }
 
-  override def anyField(value: expr, attrName: String): String =
-    s"${translate(value)}->${doName(attrName)}"
-
   override def doLocalName(s: String) = {
     s match {
-      case Identifier.ITERATOR => "$_"
-      case Identifier.ITERATOR2 => "$_buf"
-      case Identifier.INDEX => "$i"
-      case _ => s"$$this->${doName(s)}"
+      case Identifier.ITERATOR => "_"
+      case Identifier.INDEX => "i"
+      case _ => s"self.${doName(s)}"
     }
   }
 
-  override def doName(s: String) = s"${Utils.lowerCamelCase(s)}()"
+  override def doName(s: String) = s
 
   override def doEnumByLabel(enumTypeAbs: List[String], label: String): String = {
     val enumClass = types2classAbs(enumTypeAbs)
@@ -72,7 +66,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
 
   // Predefined methods of various types
   override def strConcat(left: Ast.expr, right: Ast.expr): String =
-    s"${translate(left)} . ${translate(right)}"
+    "format!(\"{}{}\", " + translate(left) + ", " + translate(right) + ")"
 
   override def strToInt(s: expr, base: expr): String =
     s"intval(${translate(s)}, ${translate(base)})"
@@ -107,25 +101,15 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
     s"${translate(s)}.substring(${translate(from)}, ${translate(to)})"
 
   override def arrayFirst(a: expr): String =
-    s"${translate(a)}[0]"
-  override def arrayLast(a: expr): String = {
-    // For huge debate on efficiency of PHP last element of array methods, see:
-    // http://stackoverflow.com/a/41795859/487064
-    val v = translate(a)
-    s"$v[count($v) - 1]"
-  }
+    s"${translate(a)}.first()"
+  override def arrayLast(a: expr): String =
+    s"${translate(a)}.last()"
   override def arraySize(a: expr): String =
-    s"count(${translate(a)})"
+    s"${translate(a)}.len()"
   override def arrayMin(a: Ast.expr): String =
-    s"min(${translate(a)})"
+    s"${translate(a)}.iter().min()"
   override def arrayMax(a: Ast.expr): String =
-    s"max(${translate(a)})"
-
-  val namespaceRef = if (config.phpNamespace.isEmpty) {
-    ""
-  } else {
-    "\\" + config.phpNamespace
-  }
+    s"${translate(a)}.iter().max()"
 
   def types2classAbs(names: List[String]) =
     names match {

From d6e6f0be37d5880225634d93ba170635407d0172 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 18:00:00 +0100
Subject: [PATCH 119/230] Fixed ternary operator to if expression

---
 .../scala/io/kaitai/struct/translators/RustTranslator.scala   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index 46fed1e4c..a33dbc75b 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -62,7 +62,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
   override def doSubscript(container: expr, idx: expr): String =
     s"${translate(container)}[${translate(idx)}]"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
-    s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
+    "if " + translate(condition) +
+    	" { " + translate(ifTrue) + " } else { " +
+	translate(ifFalse) + "}"
 
   // Predefined methods of various types
   override def strConcat(left: Ast.expr, right: Ast.expr): String =

From e088d40b2ae120b11693fe7df15f3c1862b0ef72 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 18:21:56 +0100
Subject: [PATCH 120/230] Modulo and repeat..until working

---
 .../struct/languages/RustCompiler.scala       | 21 +++++++++++--------
 .../struct/translators/RustTranslator.scala   |  7 ++++---
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index ce2fdd2ce..af4660897 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -266,24 +266,27 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
-    out.puts(s"${privateMemberName(id)} = [];")
-    out.puts("$i = 0;")
-    out.puts("do {")
+      out.puts(s"${privateMemberName(RawIdentifier(id))}: Vector<${kaitaiType2NativeType(dataType)}> = vec!();")
+    out.puts(s"${privateMemberName(id)}: Vector<${kaitaiType2NativeType(dataType)}> = vec!();")
+    out.puts("while {")
     out.inc
   }
 
   override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = {
-    val tmpName = translator.doLocalName(if (isRaw) Identifier.ITERATOR2 else Identifier.ITERATOR)
-    out.puts(s"$tmpName = $expr;")
-    out.puts(s"${privateMemberName(id)}[] = $tmpName;")
+    val tempVar = if (isRaw) {
+      translator.doName(Identifier.ITERATOR2)
+    } else {
+      translator.doName(Identifier.ITERATOR)
+    }
+    out.puts(s"$tempVar = $expr;")
+    out.puts(s"${privateMemberName(id)}.append($expr);")
   }
 
   override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
     typeProvider._currentIteratorType = Some(dataType)
-    out.puts("$i++;")
+    out.puts(s"!(${expression(untilExpr)})")
     out.dec
-    out.puts(s"} while (!(${expression(untilExpr)}));")
+    out.puts("} { }")
   }
 
   override def handleAssignmentSimple(id: Identifier, expr: String): Unit = {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index a33dbc75b..2e564a303 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -33,9 +33,9 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
   override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = {
     (detectType(left), detectType(right), op) match {
       case (_: IntType, _: IntType, Ast.operator.Div) =>
-        s"intval(${translate(left)} / ${translate(right)})"
+        s"${translate(left)} / ${translate(right)}"
       case (_: IntType, _: IntType, Ast.operator.Mod) =>
-        s"${RustCompiler.kstreamName}::mod(${translate(left)}, ${translate(right)})"
+        s"${translate(left)} % ${translate(right)}"
       case _ =>
         super.numericBinOp(left, op, right)
     }
@@ -43,7 +43,8 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
 
   override def doLocalName(s: String) = {
     s match {
-      case Identifier.ITERATOR => "_"
+      case Identifier.ITERATOR => "tmpa"
+      case Identifier.ITERATOR2 => "tmpb"
       case Identifier.INDEX => "i"
       case _ => s"self.${doName(s)}"
     }

From f4885744f004ec58b18fcf0a5ecb91bf90516198 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 18:24:59 +0100
Subject: [PATCH 121/230] Local temporary variables implemented

---
 .../scala/io/kaitai/struct/languages/RustCompiler.scala     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index af4660897..60141535b 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -274,11 +274,11 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = {
     val tempVar = if (isRaw) {
-      translator.doName(Identifier.ITERATOR2)
+      translator.doLocalName(Identifier.ITERATOR2)
     } else {
-      translator.doName(Identifier.ITERATOR)
+      translator.doLocalName(Identifier.ITERATOR)
     }
-    out.puts(s"$tempVar = $expr;")
+    out.puts(s"let $tempVar = $expr;")
     out.puts(s"${privateMemberName(id)}.append($expr);")
   }
 

From e0a6410b07c7f86341ac4d3fc95602778f869095 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 18:30:16 +0100
Subject: [PATCH 122/230] Fixed nonexistent escape characters in translator

---
 .../scala/io/kaitai/struct/translators/RustTranslator.scala | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index 2e564a303..b1fcbcd98 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -20,11 +20,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
     '\n' -> "\\n",
     '\r' -> "\\r",
     '"' -> "\\\"",
-    '\\' -> "\\\\",
-
-    '\f' -> "\\f",
-    '\13' -> "\\v",
-    '\33' -> "\\e"
+    '\\' -> "\\\\"
   )
 
   override def strLiteralUnicode(code: Char): String =

From e5b86e55e5ffc02dc4993c40da50bf1fed624e37 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 18:47:37 +0100
Subject: [PATCH 123/230] Instance methods are now in a separate impl, separate
 to the KaitaiStruct trait

---
 .../main/scala/io/kaitai/struct/RustClassCompiler.scala   | 7 +++++++
 .../scala/io/kaitai/struct/languages/RustCompiler.scala   | 8 ++++++++
 .../struct/languages/components/LanguageCompiler.scala    | 3 ++-
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
index 36dd17864..8663a1187 100644
--- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
@@ -71,6 +71,13 @@ class RustClassCompiler(
     lang.classConstructorFooter
   }
 
+  override def compileInstances(curClass: ClassSpec, extraAttrs: ListBuffer[AttrSpec]) = {
+    lang.instanceDeclHeader(curClass.name)
+    curClass.instances.foreach { case (instName, instSpec) =>
+      compileInstance(curClass.name, instName, instSpec, extraAttrs, curClass.meta.endian)
+    }
+  }
+
   override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, extraAttrs: ListBuffer[AttrSpec], endian: Option[Endianness]): Unit = {
     // FIXME: support calculated endianness
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index 60141535b..c3678cc60 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -367,6 +367,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"    pub ${idToStr(attrName)}: Option<${kaitaiType2NativeType(attrType)}>,")
   }
 
+  override def instanceDeclHeader(className: List[String]): Unit = {
+    out.dec
+    out.puts("}")
+    out.puts
+
+    out.puts(s"impl ${type2class(className)} {")
+  }
+
   override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = {
     out.puts(s"fn ${idToStr(instName)}() -> ${kaitaiType2NativeType(dataType)} {")
     out.inc
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index 77ec48c19..e1392a4cb 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -116,7 +116,8 @@ abstract class LanguageCompiler(
   def seek(io: String, pos: Ast.expr): Unit
   def popPos(io: String): Unit
   def alignToByte(io: String): Unit
-
+  
+  def instanceDeclHeader(className: List[String]): Unit = {}
   def instanceClear(instName: InstanceIdentifier): Unit = {}
   def instanceSetCalculated(instName: InstanceIdentifier): Unit = {}
   def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = attributeDeclaration(attrName, attrType, isNullable)

From 85e15edc3cddb9118099e436400130291daf5843 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 21:25:10 +0100
Subject: [PATCH 124/230] Fixed imports, and bytesToStr with ASCII encoding

---
 .../struct/languages/RustCompiler.scala       | 66 +++++++++++--------
 .../struct/translators/RustTranslator.scala   |  8 ++-
 2 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index c3678cc60..03e9226c6 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -29,40 +29,40 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override val translator: RustTranslator = new RustTranslator(typeProvider, config)
 
-  override def results(topClass: ClassSpec): Map[String, String] = {
-    val fn = topClass.nameAsStr
-    Map(
-      s"$fn.rs" -> out.result
-    )
-  }
-
   override def universalFooter: Unit = {
     out.dec
     out.puts("}")
   }
 
+  override def outImports(topClass: ClassSpec) =
+    importList.toList.map((x) => s"use $x;").mkString("", "\n", "\n")
+    
+
   override def indent: String = "    "
   override def outFileName(topClassName: String): String = s"$topClassName.rs"
 
   override def fileHeader(topClassName: String): Unit = {
-    out.puts(s"// $headerComment")
-    out.puts
-    
-    out.puts("use std::{")
-    out.puts("    option::Option,")
-    out.puts("    boxed::Box,")
-    out.puts("    io::{Result, Cursor},")
-    out.puts("    default::Default")
-    out.puts("};")
-    out.puts
+    outHeader.puts(s"// $headerComment")
+    outHeader.puts
+
+    importList.add("std::option::Option")
+    importList.add("std::boxed::Box")
+    importList.add("std::io::Result")
+    importList.add("std::io::Cursor")
+    importList.add("std::default::Default")
+    importList.add("kaitai_struct::KaitaiStream")
+    importList.add("kaitai_struct::KaitaiStruct")
 
-    out.puts("use kaitai_struct::{")
-    out.puts("    KaitaiStream,")
-    out.puts("    KaitaiStruct")
-    out.puts("};")
     out.puts
   }
 
+  override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = {
+    val name = type2class(classSpec.name.last)
+    val pkg = type2classAbs(classSpec.name)
+    
+    importList.add(s"$pkg::$name")
+  }
+
   override def classHeader(name: List[String]): Unit =
     classHeader(name, Some(kstructName))
 
@@ -192,12 +192,19 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
         }
         out.puts(s"$destName = $kstreamName::processRotateLeft($srcName, $expr, 1);")
       case ProcessCustom(name, args) =>
-        val isAbsolute = name.length > 1
-        val procClass = name.map((x) => type2class(x)).mkString(
-          if (isAbsolute) "\\" else "", "\\", ""
-        )
-        out.puts(s"let _process = $procClass::new(${args.map(expression).mkString(", ")});")
-        out.puts(s"$destName = _process.decode($srcName);")
+        val procClass = if (name.length == 1) {
+          val onlyName = name.head
+          val className = type2class(onlyName)
+          importList.add(s"$onlyName::$className")
+          className
+        } else {
+          val pkgName = name.init.mkString(".")
+          importList.add(s"$pkgName")
+          s"$pkgName.${type2class(name.last)}"
+        }
+
+        out.puts(s"let _process = $procClass::new(${args.map(expression).mkString(", ")})")
+        out.puts(s"$destName = _process.decode($srcName)")
     }
   }
 
@@ -323,6 +330,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           }
           s", $parent, ${privateMemberName(RootIdentifier)}$addEndian"
         }
+	
         s"${translator.types2classAbs(t.classSpec.get.name)}::new(stream, self, _root)"
     }
   }
@@ -373,6 +381,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts
 
     out.puts(s"impl ${type2class(className)} {")
+    out.inc
   }
 
   override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = {
@@ -510,6 +519,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
   
   def type2class(names: List[String]) = types2classRel(names)
+
+  def type2classAbs(names: List[String]) =
+    names.mkString("::")
 }
 
 object RustCompiler extends LanguageCompilerStatic
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index b1fcbcd98..faacb3911 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -89,7 +89,13 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
     }
   }
   override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String =
-    s"${RustCompiler.kstreamName}::bytesToStr($bytesExpr, ${translate(encoding)})"
+    translate(encoding) match {
+      case "ASCII" =>
+        s"String::from_utf8_lossy($bytesExpr)"
+      case _ =>
+        "panic!(\"Unimplemented encoding for bytesToStr: {}\", " +
+	translate(encoding) + ")"
+    }
   override def bytesLength(b: Ast.expr): String =
     s"strlen(${translate(b)})"
   override def strLength(s: expr): String =

From 223c7c4897362b9c7fd5cb4e5984bb8aae1a253a Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 21:30:26 +0100
Subject: [PATCH 125/230] Actually fixed ASCII encoding

---
 .../scala/io/kaitai/struct/translators/RustTranslator.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index faacb3911..4231646b8 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -90,7 +90,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
   }
   override def bytesToStr(bytesExpr: String, encoding: Ast.expr): String =
     translate(encoding) match {
-      case "ASCII" =>
+      case "\"ASCII\"" =>
         s"String::from_utf8_lossy($bytesExpr)"
       case _ =>
         "panic!(\"Unimplemented encoding for bytesToStr: {}\", " +

From a6633901fd81a7a20e11540bd59f4f87d0f23c02 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Wed, 11 Apr 2018 21:36:25 +0100
Subject: [PATCH 126/230] Fixed issue where self wasn't declared for instance
 variables

---
 .../main/scala/io/kaitai/struct/languages/RustCompiler.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index 03e9226c6..ceade45b7 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -385,7 +385,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = {
-    out.puts(s"fn ${idToStr(instName)}() -> ${kaitaiType2NativeType(dataType)} {")
+    out.puts(s"fn ${idToStr(instName)}(&mut self) -> ${kaitaiType2NativeType(dataType)} {")
     out.inc
   }
 

From 6869d23658fe5bd67cbaf7694c1e692722fa3df4 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Thu, 12 Apr 2018 10:04:14 +0100
Subject: [PATCH 127/230] Fixed intToStr for base 10

---
 .../scala/io/kaitai/struct/translators/RustTranslator.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index 4231646b8..328d19c38 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -83,7 +83,7 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
     val baseStr = translate(base)
     baseStr match {
       case "10" =>
-        s"strval(${translate(i)})"
+        s"${translate(i)}.to_string()"
       case _ =>
         s"base_convert(strval(${translate(i)}), 10, $baseStr)"
     }

From fae169f056b0e2e7068abba72349fdd6ab68f322 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Thu, 12 Apr 2018 10:31:44 +0100
Subject: [PATCH 128/230] Fixed vector typing, and integer conversions

---
 .../io/kaitai/struct/languages/RustCompiler.scala     |  9 +++++----
 .../io/kaitai/struct/translators/RustTranslator.scala | 11 ++++++++---
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index ceade45b7..ece1d4bb0 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -49,6 +49,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     importList.add("std::boxed::Box")
     importList.add("std::io::Result")
     importList.add("std::io::Cursor")
+    importList.add("std::vec::Vec")
     importList.add("std::default::Default")
     importList.add("kaitai_struct::KaitaiStream")
     importList.add("kaitai_struct::KaitaiStruct")
@@ -261,8 +262,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))}: Vector<${kaitaiType2NativeType(dataType)}> = vec!();")
-    out.puts(s"${privateMemberName(id)}: Vector<${kaitaiType2NativeType(dataType)}> = vec!();")
+      out.puts(s"${privateMemberName(RawIdentifier(id))}: Vec<${kaitaiType2NativeType(dataType)}> = vec!();")
+    out.puts(s"${privateMemberName(id)}: Vec<${kaitaiType2NativeType(dataType)}> = vec!();")
     out.puts(s"for i in 0..${expression(repeatExpr)} {")
     out.inc
   }
@@ -273,8 +274,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))}: Vector<${kaitaiType2NativeType(dataType)}> = vec!();")
-    out.puts(s"${privateMemberName(id)}: Vector<${kaitaiType2NativeType(dataType)}> = vec!();")
+      out.puts(s"${privateMemberName(RawIdentifier(id))}: Vec<${kaitaiType2NativeType(dataType)}> = vec!();")
+    out.puts(s"${privateMemberName(id)}: Vec<${kaitaiType2NativeType(dataType)}> = vec!();")
     out.puts("while {")
     out.inc
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index 328d19c38..ff17590f7 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -68,16 +68,21 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
     "format!(\"{}{}\", " + translate(left) + ", " + translate(right) + ")"
 
   override def strToInt(s: expr, base: expr): String =
-    s"intval(${translate(s)}, ${translate(base)})"
+    translate(base) match {
+      case "10" =>
+        s"${translate(s)}.parse().unwrap()"
+      case _ =>
+        "panic!(\"Converting from string to int in base {} is unimplemented\"" + translate(base) + ")"
+    }
 
   override def enumToInt(v: expr, et: EnumType): String =
     translate(v)
 
   override def boolToInt(v: expr): String =
-    s"intval(${translate(v)})"
+    s"${translate(v)} as i32"
 
   override def floatToInt(v: expr): String =
-    s"intval(${translate(v)})"
+    s"${translate(v)} as i32"
 
   override def intToStr(i: expr, base: expr): String = {
     val baseStr = translate(base)

From 406f4caef196a41c0be26482db49bdc99d98394a Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Thu, 12 Apr 2018 10:41:08 +0100
Subject: [PATCH 129/230] Fixed error with undeclared io variable in useIO, and
 error where member variables would get redeclared with a new type if it was a
 vector

---
 .../io/kaitai/struct/languages/RustCompiler.scala      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index ece1d4bb0..12416fb98 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -223,7 +223,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def useIO(ioEx: Ast.expr): String = {
-    out.puts(s"io = ${expression(ioEx)};")
+    out.puts(s"let mut io = ${expression(ioEx)};")
     "io"
   }
 
@@ -262,8 +262,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))}: Vec<${kaitaiType2NativeType(dataType)}> = vec!();")
-    out.puts(s"${privateMemberName(id)}: Vec<${kaitaiType2NativeType(dataType)}> = vec!();")
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = vec!();")
+    out.puts(s"${privateMemberName(id)} = vec!();")
     out.puts(s"for i in 0..${expression(repeatExpr)} {")
     out.inc
   }
@@ -274,8 +274,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))}: Vec<${kaitaiType2NativeType(dataType)}> = vec!();")
-    out.puts(s"${privateMemberName(id)}: Vec<${kaitaiType2NativeType(dataType)}> = vec!();")
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = vec!();")
+    out.puts(s"${privateMemberName(id)} = vec!();")
     out.puts("while {")
     out.inc
   }

From 1d814cd61d917c58322e2ec2f971502b82c710f3 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Thu, 12 Apr 2018 10:51:00 +0100
Subject: [PATCH 130/230] Fixed some issues with types and recursive types

---
 .../kaitai/struct/languages/RustCompiler.scala | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index 12416fb98..cba1e64f1 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -469,13 +469,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case _: StrType => "String"
       case _: BytesType => "Vec<u8>"
 
-      case t: UserType => types2classRel(t.name)
-      case t: EnumType => types2classRel(t.name)
+      case t: UserType => s"Box<${types2classRel(t.name)}>"
+      case t: EnumType => s"Box<${types2classRel(t.name)}"
 
       case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>"
 
-      case KaitaiStreamType => s"$kstreamName"
-      case KaitaiStructType => s"$kstructName"
+      case KaitaiStreamType => s"Option<Box<KaitaiStream>>"
+      case KaitaiStructType => s"Option<Box<KaitaiStruct>>"
       
       case SwitchType(on, cases) => kaitaiType2NativeType(TypeDetector.combineTypes(cases.values))
     }
@@ -505,14 +505,14 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case _: StrType => "\"\""
       case _: BytesType => "vec!()"
 
-      case t: UserType => ""
-      case t: EnumType => ""
+      case t: UserType => "Default::default()"
+      case t: EnumType => "Default::default()"
       // TODO: figure this out
 
-      case ArrayType(inType) => s"vec!()"
+      case ArrayType(inType) => "vec!()"
 
-      case KaitaiStreamType => s"$kstreamName"
-      case KaitaiStructType => s"$kstructName"
+      case KaitaiStreamType => "None"
+      case KaitaiStructType => "None"
       
       case SwitchType(on, cases) => ""
       // TODO

From 2f14572c2041f3bd7cfb6caef349d89c9290def7 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Thu, 12 Apr 2018 11:16:19 +0100
Subject: [PATCH 131/230] Added error checking to more IO routines, and fixed a
 missing > in generic types on Box

---
 .../io/kaitai/struct/languages/RustCompiler.scala  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index cba1e64f1..7ddd7a8f6 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -306,15 +306,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case t: ReadableType =>
         s"$io.read_${t.apiCall(defEndian)}()?"
       case blt: BytesLimitType =>
-        s"$io.read_bytes(${expression(blt.size)})"
+        s"$io.read_bytes(${expression(blt.size)})?"
       case _: BytesEosType =>
-        s"$io.read_bytes_full()"
+        s"$io.read_bytes_full()?"
       case BytesTerminatedType(terminator, include, consume, eosError, _) =>
-        s"$io.read_bytes_term($terminator, $include, $consume, $eosError)"
+        s"$io.read_bytes_term($terminator, $include, $consume, $eosError)?"
       case BitsType1 =>
-        s"$io.read_bits_int(1) != 0"
+        s"$io.read_bits_int(1)? != 0"
       case BitsType(width: Int) =>
-        s"$io.read_bits_int($width)"
+        s"$io.read_bits_int($width)?"
       case t: UserType =>
         val addParams = Utils.join(t.args.map((a) => translator.translate(a)), "", ", ", ", ")
         val addArgs = if (t.isOpaque) {
@@ -332,7 +332,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           s", $parent, ${privateMemberName(RootIdentifier)}$addEndian"
         }
 	
-        s"${translator.types2classAbs(t.classSpec.get.name)}::new(stream, self, _root)"
+        s"${translator.types2classAbs(t.classSpec.get.name)}::new(stream, self, _root)?"
     }
   }
 
@@ -470,7 +470,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case _: BytesType => "Vec<u8>"
 
       case t: UserType => s"Box<${types2classRel(t.name)}>"
-      case t: EnumType => s"Box<${types2classRel(t.name)}"
+      case t: EnumType => s"Box<${types2classRel(t.name)}>"
 
       case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>"
 

From 2ce966429e9b07cd2e3f90a895d781abd2e0ce52 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Thu, 12 Apr 2018 11:27:44 +0100
Subject: [PATCH 132/230] Wrapped new user defined types in a box when
 instantiating

---
 .../main/scala/io/kaitai/struct/languages/RustCompiler.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index 7ddd7a8f6..aab34904c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -332,7 +332,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           s", $parent, ${privateMemberName(RootIdentifier)}$addEndian"
         }
 	
-        s"${translator.types2classAbs(t.classSpec.get.name)}::new(stream, self, _root)?"
+        s"Box::new(${translator.types2classAbs(t.classSpec.get.name)}::new(stream, self, _root)?)"
     }
   }
 

From 80e0ba7fd39e66687300b8a3ebb65f08a67eb972 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 14 Apr 2018 16:16:06 +0100
Subject: [PATCH 133/230] CppTranslator: major fix for proper generation of C++
 integer literals

---
 .../struct/translators/CppTranslator.scala    | 52 +++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 18cae159a..4a6d50e8b 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -13,6 +13,58 @@ import io.kaitai.struct.languages.CppCompiler
 class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends BaseTranslator(provider) {
   val CHARSET_UTF8 = Charset.forName("UTF-8")
 
+  val MAX_UINT64 = BigInt("18446744073709551615")
+
+  /**
+    * Handles integer literals for C++ by appending relevant suffix to
+    * decimal notation.
+    *
+    * Note that suffixes essentially mean "long", "unsigned long",
+    * and "unsigned long long", which are not really guaranteed to match
+    * `int32_t`, `uint32_t` and `uint64_t`, but it would work for majority
+    * of current compilers.
+    *
+    * For reference, ranges of integers that are used in this conversion are:
+    *
+    * * int32_t (no suffix): –2147483648..2147483647
+    * * uint32_t (UL): 0..4294967295
+    * * int64_t (LL): -9223372036854775808..9223372036854775807
+    * * uint64_t (ULL): 0..18446744073709551615
+    *
+    * Merging all these ranges, we get the following decision tree:
+    *
+    * * -9223372036854775808..-2147483649 => LL
+    * * -2147483648..2147483647 => no suffix
+    * * 2147483648..4294967295 => UL
+    * * 4294967296..9223372036854775807 => LL
+    * * 9223372036854775808..18446744073709551615 => ULL
+    *
+    * Beyond these boundaries, C++ is unlikely to be able to represent
+    * these anyway, so we just drop the suffix and hope for the miracle.
+    *
+    * @param n integer to render
+    * @return rendered integer literal in C++ syntax as string
+    */
+  override def doIntLiteral(n: BigInt): String = {
+    val suffix = if (n < -9223372036854775808L) {
+      "" // too low, no suffix would help anyway
+    } else if (n <= -2147483649L) {
+      "LL" // -9223372036854775808..–2147483649
+    } else if (n <= 2147483647L) {
+      "" // -2147483648..2147483647
+    } else if (n <= 4294967295L) {
+      "UL" // 2147483648..4294967295
+    } else if (n <= 9223372036854775807L) {
+      "LL" // 4294967296..9223372036854775807
+    } else if (n <= MAX_UINT64) {
+      "ULL" // 9223372036854775808..18446744073709551615
+    } else {
+      "" // too high, no suffix would help anyway
+    }
+
+    s"$n$suffix"
+  }
+
   /**
     * Handles string literal for C++ by wrapping a C `const char*`-style string
     * into a std::string constructor. Note that normally std::string

From 44422383078570706bc9234713a6c9759d16ee4c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 15 Apr 2018 00:21:10 +0100
Subject: [PATCH 134/230] GoCompiler: implemented fixed contents

---
 .../kaitai/struct/languages/GoCompiler.scala  | 24 +++++++++++++++----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index 430ff940b..b4ee80f02 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -1,7 +1,7 @@
 package io.kaitai.struct.languages
 
-import io.kaitai.struct.datatype.{DataType, FixedEndian}
 import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.datatype.{DataType, FixedEndian}
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
@@ -16,8 +16,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     with UniversalFooter
     with UniversalDoc
     with AllocateIOLocalVar
-    with GoReads
-    with FixedContentsUsingArrayByteLiteral {
+    with GoReads {
   import GoCompiler._
 
   override val translator = new GoTranslator(out, typeProvider, importList)
@@ -117,8 +116,23 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit = ???
 
-  override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit = {
-    out.puts(s"${privateMemberName(attrName)} = $normalIO.ensureFixedContents($contents);")
+  override def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]): Unit = {
+    out.puts(s"${privateMemberName(attrName)}, err = $normalIO.ReadBytes(${contents.length})")
+
+    out.puts(s"if err != nil {")
+    out.inc
+    out.puts("return err")
+    out.dec
+    out.puts("}")
+
+    importList.add("bytes")
+    importList.add("errors")
+    val expected = translator.resToStr(translator.doByteArrayLiteral(contents))
+    out.puts(s"if !bytes.Equal(${privateMemberName(attrName)}, $expected) {")
+    out.inc
+    out.puts("return errors.New(\"Unexpected fixed contents\")")
+    out.dec
+    out.puts("}")
   }
 
   override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = {

From 5533915462cba5987e8907968639321b1cc9d20a Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 15 Apr 2018 00:41:01 +0100
Subject: [PATCH 135/230] Moved MAX_UINT64 const to Utils

---
 shared/src/main/scala/io/kaitai/struct/Utils.scala     |  5 +++++
 .../io/kaitai/struct/translators/CppTranslator.scala   | 10 ++++------
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/Utils.scala b/shared/src/main/scala/io/kaitai/struct/Utils.scala
index a14816f37..f32c50e2f 100644
--- a/shared/src/main/scala/io/kaitai/struct/Utils.scala
+++ b/shared/src/main/scala/io/kaitai/struct/Utils.scala
@@ -5,6 +5,11 @@ import java.nio.charset.Charset
 import scala.collection.mutable.ListBuffer
 
 object Utils {
+  /**
+    * BigInt-typed max value of unsigned 64-bit integer.
+    */
+  val MAX_UINT64 = BigInt("18446744073709551615")
+
   private val RDecimal = "^(-?[0-9]+)$".r
   private val RHex = "^0x([0-9a-fA-F]+)$".r
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 4a6d50e8b..130f7e967 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -2,19 +2,17 @@ package io.kaitai.struct.translators
 
 import java.nio.charset.Charset
 
-import io.kaitai.struct.{ImportList, Utils}
-import io.kaitai.struct.exprlang.Ast
-import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.exprlang.Ast
+import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.format.Identifier
 import io.kaitai.struct.languages.CppCompiler
+import io.kaitai.struct.{ImportList, Utils}
 
 class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends BaseTranslator(provider) {
   val CHARSET_UTF8 = Charset.forName("UTF-8")
 
-  val MAX_UINT64 = BigInt("18446744073709551615")
-
   /**
     * Handles integer literals for C++ by appending relevant suffix to
     * decimal notation.
@@ -56,7 +54,7 @@ class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends B
       "UL" // 2147483648..4294967295
     } else if (n <= 9223372036854775807L) {
       "LL" // 4294967296..9223372036854775807
-    } else if (n <= MAX_UINT64) {
+    } else if (n <= Utils.MAX_UINT64) {
       "ULL" // 9223372036854775808..18446744073709551615
     } else {
       "" // too high, no suffix would help anyway

From fc0c2ee308947a241f54b5317fb2e057362ccc44 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 15 Apr 2018 00:41:22 +0100
Subject: [PATCH 136/230] GoTranslator: implemented type-aware integer
 literals, C++-style

---
 .../struct/translators/GoTranslator.scala      | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 29fe050c8..2a51f351f 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -105,6 +105,24 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   def trStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult =
     ResultString(doStrCompareOp(left, op, right))
 
+  override def doIntLiteral(n: BigInt): String = {
+    if (n < -9223372036854775808L) {
+      s"$n" // too low, no type conversion would help anyway
+    } else if (n <= -2147483649L) {
+      s"int64($n)" // -9223372036854775808..–2147483649
+    } else if (n <= 2147483647L) {
+      s"$n" // -2147483648..2147483647
+    } else if (n <= 4294967295L) {
+      s"uint32($n)" // 2147483648..4294967295
+    } else if (n <= 9223372036854775807L) {
+      s"int64($n)" // 4294967296..9223372036854775807
+    } else if (n <= Utils.MAX_UINT64) {
+      s"uint64($n)" // 9223372036854775808..18446744073709551615
+    } else {
+      s"$n" // too high, no type conversion would help anyway
+    }
+  }
+
   override def unaryOp(op: Ast.unaryop): String = op match {
     case Ast.unaryop.Invert => "^"
     case Ast.unaryop.Minus => "-"

From 753565908f53ccd2b3cc43a0ee071cbc189b24a6 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 15 Apr 2018 14:51:14 +0100
Subject: [PATCH 137/230] GoTranslator: properly implemented mod

---
 .../struct/translators/GoTranslator.scala     | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 2a51f351f..461a2a27d 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -93,8 +93,21 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   def trBooleanOp(op: Ast.boolop, values: Seq[Ast.expr]) =
     ResultString(doBooleanOp(op, values))
 
-  def trNumericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) =
-    ResultString(numericBinOp(left, op, right))
+  def trNumericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr): TranslatorResult = {
+    (detectType(left), detectType(right), op) match {
+      case (t1: IntType, t2: IntType, Ast.operator.Mod) =>
+        val v1 = allocateLocalVar()
+        out.puts(s"${localVarName(v1)} := ${translate(left)} % ${translate(right)}")
+        out.puts(s"if ${localVarName(v1)} < 0 {")
+        out.inc
+        out.puts(s"${localVarName(v1)} += ${translate(right)}")
+        out.dec
+        out.puts("}")
+        ResultLocalVar(v1)
+      case _ =>
+        ResultString(numericBinOp(left, op, right))
+    }
+  }
 
   def trStrConcat(left: Ast.expr, right: Ast.expr): TranslatorResult =
     ResultString(translate(left) + " + " + translate(right))
@@ -129,15 +142,6 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     case Ast.unaryop.Not => "!"
   }
 
-  override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = {
-    (detectType(left), detectType(right), op) match {
-      case (_: IntType, _: IntType, Ast.operator.Mod) =>
-        s"${GoCompiler.kstreamName}.mod(${translate(left)}, ${translate(right)})"
-      case _ =>
-        super.numericBinOp(left, op, right)
-    }
-  }
-
   def trLocalName(s: String): TranslatorResult = {
     s match {
       case Identifier.ROOT |

From 0af4bb49e5609394e1511af3aaf4fd7a542e067d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 15 Apr 2018 21:15:07 +0100
Subject: [PATCH 138/230] GoCompiler: implemented repeat-until properly

---
 .../kaitai/struct/languages/GoCompiler.scala  | 21 +++++++------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index b4ee80f02..af44be1aa 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -234,29 +234,24 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
     if (needRaw)
       out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>();")
-    out.puts(s"${privateMemberName(id)} = new ${kaitaiType2NativeType(ArrayType(dataType))}();")
-    out.puts("{")
-    out.inc
-    out.puts(s"${kaitaiType2NativeType(dataType)} ${translator.specialName(Identifier.ITERATOR)};")
-    out.puts("do {")
+    out.puts("for {")
     out.inc
   }
 
   override def handleAssignmentRepeatUntil(id: Identifier, r: TranslatorResult, isRaw: Boolean): Unit = {
     val expr = translator.resToStr(r)
-    val (typeDecl, tempVar) = if (isRaw) {
-      ("byte[] ", translator.specialName(Identifier.ITERATOR2))
-    } else {
-      ("", translator.specialName(Identifier.ITERATOR))
-    }
-    out.puts(s"$typeDecl$tempVar = $expr;")
-    out.puts(s"${privateMemberName(id)}.add($tempVar);")
+    val tempVar = translator.specialName(Identifier.ITERATOR)
+    out.puts(s"$tempVar := $expr")
+    out.puts(s"${privateMemberName(id)} = append(${privateMemberName(id)}, $tempVar)")
   }
 
   override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
     typeProvider._currentIteratorType = Some(dataType)
+    out.puts(s"if ${expression(untilExpr)} {")
+    out.inc
+    out.puts("break")
     out.dec
-    out.puts(s"} while (!(${expression(untilExpr)}));")
+    out.puts("}")
     out.dec
     out.puts("}")
   }

From b6af6826d8771ab6a52f74525319664ab11f690c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 16 Apr 2018 01:23:24 +0100
Subject: [PATCH 139/230] Go: fixed EOF and read-eos handling

---
 .../io/kaitai/struct/languages/GoCompiler.scala     | 11 ++++++++++-
 .../struct/languages/components/GoReads.scala       |  2 +-
 .../io/kaitai/struct/translators/GoTranslator.scala | 13 +++++--------
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index af44be1aa..3573fdfaa 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -207,8 +207,17 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     if (needRaw)
       out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>();")
     //out.puts(s"${privateMemberName(id)} = make(${kaitaiType2NativeType(ArrayType(dataType))})")
-    out.puts(s"for !$io.EOF() {")
+    out.puts(s"for {")
     out.inc
+
+    val eofVar = translator.allocateLocalVar()
+    out.puts(s"${translator.localVarName(eofVar)}, err := this._io.EOF()")
+    translator.outAddErrCheck()
+    out.puts(s"if ${translator.localVarName(eofVar)} {")
+    out.inc
+    out.puts("break")
+    out.dec
+    out.puts("}")
   }
 
   override def handleAssignmentRepeatEos(id: Identifier, r: TranslatorResult): Unit = {
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
index 876515960..064de8a7a 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
@@ -1,8 +1,8 @@
 package io.kaitai.struct.languages.components
 
 import io.kaitai.struct.Utils
-import io.kaitai.struct.datatype.{BigEndian, DataType, FixedEndian}
 import io.kaitai.struct.datatype.DataType._
+import io.kaitai.struct.datatype.{DataType, FixedEndian}
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
 import io.kaitai.struct.translators.{GoTranslator, TranslatorResult}
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 461a2a27d..6a308a7cc 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -323,20 +323,17 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     ResultString(s"strconv.FormatInt(int64(${translate(value)}), ${translate(base)})")
   }
 
-  override def floatToInt(value: Ast.expr): TranslatorResult =
+  override def floatToInt(value: Ast.expr) =
     ResultString(s"int(${translate(value)})")
 
-  override def kaitaiStreamSize(value: Ast.expr): TranslatorResult = {
+  override def kaitaiStreamSize(value: Ast.expr) =
     outVarCheckRes(s"${translate(value)}.Size()")
-  }
 
-  override def kaitaiStreamEof(value: Ast.expr): TranslatorResult = {
-    ResultString(s"${translate(value)}.EOF()")
-  }
+  override def kaitaiStreamEof(value: Ast.expr) =
+    outVarCheckRes(s"${translate(value)}.EOF()")
 
-  override def kaitaiStreamPos(value: Ast.expr): TranslatorResult = {
+  override def kaitaiStreamPos(value: Ast.expr) =
     outVarCheckRes(s"${translate(value)}.Pos()")
-  }
 
   override def arrayMin(a: Ast.expr): ResultLocalVar = {
     val min = allocateLocalVar()

From ee945dbcf3656947f90e4af757be1fd758ae2c6b Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 17 Apr 2018 13:13:35 +0100
Subject: [PATCH 140/230] Go: implemented basic enum support

---
 .../io/kaitai/struct/GoClassCompiler.scala    |  5 +-
 .../kaitai/struct/languages/GoCompiler.scala  | 58 ++++++-------------
 .../struct/languages/components/GoReads.scala |  8 ++-
 .../struct/translators/GoTranslator.scala     | 11 ++--
 4 files changed, 30 insertions(+), 52 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
index 0e57e55bb..733dc1577 100644
--- a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
@@ -27,6 +27,9 @@ class GoClassCompiler(
     if (!curClass.doc.isEmpty)
       lang.classDoc(curClass.name, curClass.doc)
 
+    // Enums declaration defines types, so they need to go first
+    compileEnums(curClass)
+
     // Basic struct declaration
     lang.classHeader(curClass.name)
     compileAttrDeclarations(curClass.seq ++ extraAttrs)
@@ -42,8 +45,6 @@ class GoClassCompiler(
 
     compileAttrReaders(curClass.seq ++ extraAttrs)
 
-    compileEnums(curClass)
-
     // Recursive types
     compileSubclasses(curClass)
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index 3573fdfaa..9f14bbc7d 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -315,18 +315,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"switch (${expression(on)}) {")
 
   override def switchCaseStart(condition: Ast.expr): Unit = {
-    // Java is very specific about what can be used as "condition" in "case
-    // condition:".
-    val condStr = condition match {
-      case Ast.expr.EnumByLabel(enumName, enumVal) =>
-        // If switch is over a enum, only literal enum values are supported,
-        // and they must be written as "MEMBER", not "SomeEnum.MEMBER".
-        value2Const(enumVal.name)
-      case _ =>
-        expression(condition)
-    }
-
-    out.puts(s"case $condStr: {")
+    out.puts(s"case ${expression(condition)}: {")
     out.inc
   }
 
@@ -384,42 +373,22 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"this.${calculatedFlagForName(instName)} = true")
 
   override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = {
-    val enumClass = type2class(enumName)
+    val fullEnumName: List[String] = curClass ++ List(enumName)
+    val fullEnumNameStr = types2class(fullEnumName)
 
     out.puts
-    out.puts(s"public enum $enumClass {")
+    out.puts(s"type $fullEnumNameStr int")
+    out.puts("const (")
     out.inc
 
-    if (enumColl.size > 1) {
-      enumColl.dropRight(1).foreach { case (id, label) =>
-        out.puts(s"${value2Const(label.name)}($id),")
-      }
-    }
-    enumColl.last match {
-      case (id, label) =>
-        out.puts(s"${value2Const(label.name)}($id);")
+    enumColl.foreach { case (id, label) =>
+      out.puts(s"${enumToStr(fullEnumName, label.name)} $fullEnumNameStr = $id")
     }
 
-    out.puts
-    out.puts("private final long id;")
-    out.puts(s"$enumClass(long id) { this.id = id; }")
-    out.puts("public long id() { return id; }")
-    out.puts(s"private static final Map<Long, $enumClass> byId = new HashMap<Long, $enumClass>(${enumColl.size});")
-    out.puts("static {")
-    out.inc
-    out.puts(s"for ($enumClass e : $enumClass.values())")
-    out.inc
-    out.puts(s"byId.put(e.id(), e);")
-    out.dec
     out.dec
-    out.puts("}")
-    out.puts(s"public static $enumClass byId(long id) { return byId.get(id); }")
-    out.dec
-    out.puts("}")
+    out.puts(")")
   }
 
-  def value2Const(s: String) = s.toUpperCase
-
   def idToStr(id: Identifier): String = {
     id match {
       case SpecialIdentifier(name) => name
@@ -497,7 +466,7 @@ object GoCompiler extends LanguageCompilerStatic
         case Some(cs) => cs.name
         case None => t.name
       })
-      case EnumType(name, _) => types2class(name)
+      case t: EnumType => types2class(t.enumSpec.get.name)
 
       case ArrayType(inType) => s"[]${kaitaiType2NativeType(inType)}"
 
@@ -507,6 +476,15 @@ object GoCompiler extends LanguageCompilerStatic
 
   def types2class(names: List[String]) = names.map(x => type2class(x)).mkString("_")
 
+  def enumToStr(enumTypeAbs: List[String]): String = {
+    val enumName = enumTypeAbs.last
+    val enumClass: List[String] = enumTypeAbs.dropRight(1)
+    enumToStr(enumClass, enumName)
+  }
+
+  def enumToStr(typeName: List[String], enumName: String): String =
+    types2class(typeName) + "__" + type2class(enumName)
+
   override def kstreamName: String = "kaitai.Stream"
   override def kstructName: String = "interface{}"
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
index 064de8a7a..d15b456dd 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
@@ -35,9 +35,11 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with SwitchOps {
         val r1 = translator.outVarCheckRes(parseExprBytes(t.bytes, io))
         val expr = translator.bytesToStr(translator.resToStr(r1), Ast.expr.Str(t.encoding))
         handleAssignment(id, expr, rep, isRaw)
-//      case t: EnumType =>
-//        val expr = translator.doEnumById(t.enumSpec.get.name, parseExpr(t.basedOn, io))
-//        handleAssignment(id, expr, rep, isRaw)
+      case t: EnumType =>
+        val r1 = translator.outVarCheckRes(parseExpr(t.basedOn, io, defEndian))
+        val enumSpec = t.enumSpec.get
+        val expr = translator.trEnumById(enumSpec.name, translator.resToStr(r1))
+        handleAssignment(id, expr, rep, isRaw)
       case _ =>
         val expr = parseExpr(dataType, io, defEndian)
         val r = translator.outVarCheckRes(expr)
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 6a308a7cc..296000d22 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -20,6 +20,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   with CommonArraysAndCast[TranslatorResult]
   with CommonMethods[TranslatorResult] {
 
+  import io.kaitai.struct.languages.GoCompiler._
+
   var returnRes: Option[String] = None
 
   override def translate(v: Ast.expr): String = resToStr(translateExpr(v))
@@ -193,13 +195,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
 
 //  override def doEnumByLabel(enumTypeAbs: List[String], label: String): String =
 //    s"${enumClass(enumTypeAbs)}.${label.toUpperCase}"
-//  override def doEnumById(enumTypeAbs: List[String], id: String): String =
-//    s"${enumClass(enumTypeAbs)}.byId($id)"
-
-  def enumClass(enumTypeAbs: List[String]): String = {
-    val enumTypeRel = Utils.relClass(enumTypeAbs, provider.nowClass.name)
-    enumTypeRel.map((x) => Utils.upperCamelCase(x)).mkString(".")
-  }
+  def trEnumById(enumTypeAbs: List[String], id: String) =
+    ResultString(s"${types2class(enumTypeAbs)}($id)")
 
   override def doBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): String = {
     op match {

From b24757616971154780bb9ca2d6b023d6bd405728 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 19 Apr 2018 11:10:11 +0100
Subject: [PATCH 141/230] GoTranslator: implemented byte array comparsion

---
 .../io/kaitai/struct/translators/GoTranslator.scala  | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 296000d22..40766ded5 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -67,6 +67,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
             trNumericCompareOp(left, op, right)
           case (_: StrType, _: StrType) =>
             trStrCompareOp(left, op, right)
+          case (_: BytesType, _: BytesType) =>
+            trBytesCompareOp(left, op, right)
           case (ltype, rtype) =>
             throw new TypeMismatchError(s"can't do $ltype $op $rtype")
         }
@@ -120,6 +122,16 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   def trStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult =
     ResultString(doStrCompareOp(left, op, right))
 
+  def trBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult = {
+    importList.add("bytes")
+    op match {
+      case Ast.cmpop.Eq =>
+        ResultString(s"bytes.Equal(${translate(left)}, ${translate(right)})")
+      case _ =>
+        ResultString(s"(bytes.Compare(${translate(left)}, ${translate(right)}) ${cmpOp(op)} 0)")
+    }
+  }
+
   override def doIntLiteral(n: BigInt): String = {
     if (n < -9223372036854775808L) {
       s"$n" // too low, no type conversion would help anyway

From 46410c302c6f07dc2920ec35531398633ac27957 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 19 Apr 2018 11:10:11 +0100
Subject: [PATCH 142/230] GoTranslator: implemented byte array comparsion

---
 .../io/kaitai/struct/translators/GoTranslator.scala  | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 296000d22..40766ded5 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -67,6 +67,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
             trNumericCompareOp(left, op, right)
           case (_: StrType, _: StrType) =>
             trStrCompareOp(left, op, right)
+          case (_: BytesType, _: BytesType) =>
+            trBytesCompareOp(left, op, right)
           case (ltype, rtype) =>
             throw new TypeMismatchError(s"can't do $ltype $op $rtype")
         }
@@ -120,6 +122,16 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
   def trStrCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult =
     ResultString(doStrCompareOp(left, op, right))
 
+  def trBytesCompareOp(left: Ast.expr, op: Ast.cmpop, right: Ast.expr): TranslatorResult = {
+    importList.add("bytes")
+    op match {
+      case Ast.cmpop.Eq =>
+        ResultString(s"bytes.Equal(${translate(left)}, ${translate(right)})")
+      case _ =>
+        ResultString(s"(bytes.Compare(${translate(left)}, ${translate(right)}) ${cmpOp(op)} 0)")
+    }
+  }
+
   override def doIntLiteral(n: BigInt): String = {
     if (n < -9223372036854775808L) {
       s"$n" // too low, no type conversion would help anyway

From e6b4003b20d47937253d533f6c60d50a71e7aec8 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Fri, 20 Apr 2018 15:12:55 +0100
Subject: [PATCH 143/230] Fixed struct member naming issue, and issue with
 using stream in attribute readers

---
 .../io/kaitai/struct/RustClassCompiler.scala  |  1 -
 .../struct/languages/RustCompiler.scala       | 21 +++++++++++++------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
index 8663a1187..7de42a0c7 100644
--- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
@@ -38,7 +38,6 @@ class RustClassCompiler(
     // Constructor = Read() function
     compileReadFunction(curClass, extraAttrs)
     
-
     compileInstances(curClass, extraAttrs)
 
     compileAttrReaders(curClass.seq ++ extraAttrs)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index aab34904c..f50e98851 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -36,7 +36,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def outImports(topClass: ClassSpec) =
     importList.toList.map((x) => s"use $x;").mkString("", "\n", "\n")
-    
 
   override def indent: String = "    "
   override def outFileName(topClassName: String): String = s"$topClassName.rs"
@@ -100,6 +99,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"let mut s: Self = Default::default();")
     out.puts
 
+    out.puts(s"s.stream = stream;")
+
     out.puts(s"s.read(stream, _parent, _root)?;")
     out.puts
 
@@ -139,6 +140,8 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     attrName match {
       case ParentIdentifier | RootIdentifier | IoIdentifier =>
         // just ignore it for now
+      case IoIdentifier =>
+        out.puts(s"     stream: ${kaitaiType2NativeType(attrType)},")
       case _ =>
         out.puts(s"    pub ${idToStr(attrName)}: ${kaitaiType2NativeType(attrType)},")
     }
@@ -332,7 +335,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           s", $parent, ${privateMemberName(RootIdentifier)}$addEndian"
         }
 	
-        s"Box::new(${translator.types2classAbs(t.classSpec.get.name)}::new(stream, self, _root)?)"
+        s"Box::new(${translator.types2classAbs(t.classSpec.get.name)}::new(self.stream, self, _root)?)"
     }
   }
 
@@ -432,7 +435,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def privateMemberName(id: Identifier): String = {
     id match {
-      case IoIdentifier => s"stream"
+      case IoIdentifier => s"self.stream"
       case RootIdentifier => s"_root"
       case ParentIdentifier => s"_parent"
       case _ => s"self.${idToStr(id)}"
@@ -469,8 +472,15 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case _: StrType => "String"
       case _: BytesType => "Vec<u8>"
 
-      case t: UserType => s"Box<${types2classRel(t.name)}>"
-      case t: EnumType => s"Box<${types2classRel(t.name)}>"
+      case t: UserType => t.classSpec match {
+        case Some(cs) => s"Box<${type2class(cs.name)}>"
+	case None => s"Box<${type2class(t.name)}>"
+      }
+      
+      case t: EnumType => t.enumSpec match {
+        case Some(cs) => s"Box<${type2class(cs.name)}>"
+	case None => s"Box<${type2class(t.name)}>"
+      }
 
       case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>"
 
@@ -507,7 +517,6 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
       case t: UserType => "Default::default()"
       case t: EnumType => "Default::default()"
-      // TODO: figure this out
 
       case ArrayType(inType) => "vec!()"
 

From 7da6d94dbd39c913ed6f456f081daec5576cb393 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 20 Apr 2018 23:12:44 +0100
Subject: [PATCH 144/230] Go: adjustments for better bits integers &
 boolean-based-on-bits-integers support

---
 .../main/scala/io/kaitai/struct/languages/GoCompiler.scala | 6 +++---
 .../io/kaitai/struct/languages/components/GoReads.scala    | 7 ++++++-
 .../scala/io/kaitai/struct/translators/GoTranslator.scala  | 2 ++
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index 9f14bbc7d..9421e2b59 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -172,8 +172,8 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def useIO(ioEx: Ast.expr): String = {
-    out.puts(s"$kstreamName io = ${expression(ioEx)};")
-    "io"
+    out.puts(s"thisIo := ${expression(ioEx)}")
+    "thisIo"
   }
 
   override def pushPos(io: String): Unit = {
@@ -281,7 +281,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case BytesTerminatedType(terminator, include, consume, eosError, _) =>
         s"$io.ReadBytesTerm($terminator, $include, $consume, $eosError)"
       case BitsType1 =>
-        s"$io.ReadBitsInt(1) != 0"
+        s"$io.ReadBitsInt(1)"
       case BitsType(width: Int) =>
         s"$io.ReadBitsInt($width)"
       case t: UserType =>
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
index d15b456dd..00729fd19 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
@@ -5,7 +5,7 @@ import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.datatype.{DataType, FixedEndian}
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
-import io.kaitai.struct.translators.{GoTranslator, TranslatorResult}
+import io.kaitai.struct.translators.{GoTranslator, ResultString, TranslatorResult}
 
 import scala.collection.mutable.ListBuffer
 
@@ -40,6 +40,11 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with SwitchOps {
         val enumSpec = t.enumSpec.get
         val expr = translator.trEnumById(enumSpec.name, translator.resToStr(r1))
         handleAssignment(id, expr, rep, isRaw)
+      case BitsType1 =>
+        val expr = parseExpr(dataType, io, defEndian)
+        val r1 = translator.outVarCheckRes(expr)
+        val r2 = ResultString(s"${translator.resToStr(r1)} != 0")
+        handleAssignment(id, r2, rep, isRaw)
       case _ =>
         val expr = parseExpr(dataType, io, defEndian)
         val r = translator.outVarCheckRes(expr)
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 40766ded5..50312756a 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -69,6 +69,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
             trStrCompareOp(left, op, right)
           case (_: BytesType, _: BytesType) =>
             trBytesCompareOp(left, op, right)
+          case (_: BooleanType, _: BooleanType) =>
+            trNumericCompareOp(left, op, right)
           case (ltype, rtype) =>
             throw new TypeMismatchError(s"can't do $ltype $op $rtype")
         }

From 52423942dee618e7c48e51cb3c003283ea76b897 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 21 Apr 2018 21:56:53 +0100
Subject: [PATCH 145/230] Expression: proper parsing of deeply nested enums

---
 .../io/kaitai/struct/exprlang/Expressions.scala    | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
index ae5d9f1d9..bc80f454c 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
@@ -160,8 +160,18 @@ object Expressions {
 
   val testlist1: P[Seq[Ast.expr]] = P( test.rep(1, sep = ",") )
 
-  val enumByName: P[Ast.expr.EnumByLabel] = P( (NAME) ~ "::" ~ (NAME) ).map {
-    case(enumName, enumLabel) => Ast.expr.EnumByLabel(enumName, enumLabel)
+  val enumByName: P[Ast.expr.EnumByLabel] = P("::".!.? ~ NAME.rep(2, "::")).map {
+    case (first, names: Seq[Ast.identifier]) =>
+      val isAbsolute = first.nonEmpty
+      val (enumName, enumLabel) = names.takeRight(2) match {
+        case Seq(a, b) => (a, b)
+      }
+      val typePath = names.dropRight(2)
+      if (typePath.isEmpty) {
+        Ast.expr.EnumByLabel(enumName, enumLabel, Ast.EmptyTypeId)
+      } else {
+        Ast.expr.EnumByLabel(enumName, enumLabel, Ast.typeId(isAbsolute, typePath.map(_.name)))
+      }
   }
 
   val topExpr: P[Ast.expr] = P( test ~ End )

From a68fc0b75987b93f852d46f6d62e53f14fc750a8 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 22 Apr 2018 11:32:39 +0100
Subject: [PATCH 146/230] Make translators actually use deeply nested enum
 specs

---
 .../struct/translators/TranslatorSpec.scala       |  2 +-
 .../io/kaitai/struct/ClassTypeProvider.scala      | 15 +++++++++++----
 .../io/kaitai/struct/GraphvizClassCompiler.scala  |  2 +-
 .../scala/io/kaitai/struct/exprlang/Ast.scala     |  2 +-
 .../scala/io/kaitai/struct/format/EnumSpec.scala  |  6 ++++++
 .../struct/translators/BaseTranslator.scala       | 14 ++++++++------
 .../kaitai/struct/translators/TypeDetector.scala  | 14 ++++++++------
 .../kaitai/struct/translators/TypeProvider.scala  |  2 +-
 8 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index b1ca1db9e..f8000d53a 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -607,7 +607,7 @@ class TranslatorSpec extends FunSuite {
   abstract class FakeTypeProvider extends TypeProvider {
     val nowClass = ClassSpec.opaquePlaceholder(List("top_class"))
 
-    override def resolveEnum(enumName: String) =
+    override def resolveEnum(inType: Ast.typeId, enumName: String) =
       throw new NotImplementedError
 
     override def resolveType(typeName: Ast.typeId): DataType =
diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
index ce63c7e38..9f36a0f7f 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
@@ -69,7 +69,8 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends
     }
   }
 
-  override def resolveEnum(enumName: String): EnumSpec = resolveEnum(nowClass, enumName)
+  override def resolveEnum(inType: Ast.typeId, enumName: String): EnumSpec =
+    resolveEnum(resolveClassSpec(inType), enumName)
 
   def resolveEnum(inClass: ClassSpec, enumName: String): EnumSpec = {
     inClass.enums.get(enumName) match {
@@ -86,12 +87,18 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends
   }
 
   override def resolveType(typeName: Ast.typeId): DataType =
-    resolveType(if (typeName.absolute) topClass else nowClass, typeName.names)
+    makeUserType(resolveClassSpec(typeName))
 
-  def resolveType(inClass: ClassSpec, typeName: Seq[String]): DataType =
-    makeUserType(resolveClassSpec(inClass, typeName))
+  def resolveClassSpec(typeName: Ast.typeId): ClassSpec =
+    resolveClassSpec(
+      if (typeName.absolute) topClass else nowClass,
+      typeName.names
+    )
 
   def resolveClassSpec(inClass: ClassSpec, typeName: Seq[String]): ClassSpec = {
+    if (typeName.isEmpty)
+      return inClass
+
     val headTypeName :: restTypesNames = typeName.toList
     val nextClass = resolveClassSpec(inClass, headTypeName)
     if (restTypesNames.isEmpty) {
diff --git a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
index ff172c7e4..83d7b725f 100644
--- a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
@@ -309,7 +309,7 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
         List()
       case _: expr.EnumByLabel =>
         List()
-      case expr.EnumById(enumName, id) =>
+      case expr.EnumById(_, id, _) =>
         affectedVars(id)
       case expr.Attribute(value, attr) =>
         val targetClass = translator.detectType(value)
diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
index a85b2edac..94131177e 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Ast.scala
@@ -40,7 +40,7 @@ object Ast {
     case class Str(s: String) extends expr
     case class Bool(n: Boolean) extends expr
     case class EnumByLabel(enumName: identifier, label: identifier, inType: typeId = EmptyTypeId) extends expr
-    case class EnumById(enumName: identifier, id: expr) extends expr
+    case class EnumById(enumName: identifier, id: expr, inType: typeId = EmptyTypeId) extends expr
 
     case class Attribute(value: expr, attr: identifier) extends expr
     case class CastToType(value: expr, typeName: typeId) extends expr
diff --git a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala
index 8f10e19f2..2794e0b85 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/EnumSpec.scala
@@ -3,6 +3,12 @@ package io.kaitai.struct.format
 case class EnumSpec(map: Map[Long, EnumValueSpec]) {
   var name = List[String]()
 
+  /**
+    * @return Absolute name of enum as string, components separated by
+    *         double colon operator `::`
+    */
+  def nameAsStr = name.mkString("::")
+
   /**
     * Stabilize order of generated enums by sorting it by integer ID - it
     * both looks nicer and doesn't screw diffs in generated code.
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 16fe3b633..6aca17e93 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -54,11 +54,11 @@ abstract class BaseTranslator(val provider: TypeProvider)
         doStringLiteral(s)
       case Ast.expr.Bool(n) =>
         doBoolLiteral(n)
-      case Ast.expr.EnumById(enumType, id) =>
-        val enumSpec = provider.resolveEnum(enumType.name)
+      case Ast.expr.EnumById(enumType, id, inType) =>
+        val enumSpec = provider.resolveEnum(inType, enumType.name)
         doEnumById(enumSpec.name, translate(id))
       case Ast.expr.EnumByLabel(enumType, label, inType) =>
-        val enumSpec = provider.resolveEnum(enumType.name)
+        val enumSpec = provider.resolveEnum(inType, enumType.name)
         doEnumByLabel(enumSpec.name, label.name)
       case Ast.expr.Name(name: Ast.identifier) =>
         doLocalName(name.name)
@@ -85,9 +85,11 @@ abstract class BaseTranslator(val provider: TypeProvider)
             doStrCompareOp(left, op, right)
           case (_: BytesType, _: BytesType) =>
             doBytesCompareOp(left, op, right)
-          case (EnumType(ltype, _), EnumType(rtype, _)) =>
-            if (ltype != rtype) {
-              throw new TypeMismatchError(s"can't compare enums type $ltype and $rtype")
+          case (et1: EnumType, et2: EnumType) =>
+            val et1Spec = et1.enumSpec.get
+            val et2Spec = et2.enumSpec.get
+            if (et1Spec != et2Spec) {
+              throw new TypeMismatchError(s"can't compare enums type ${et1Spec.nameAsStr} and ${et2Spec.nameAsStr}")
             } else {
               doEnumCompareOp(left, op, right)
             }
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index 5c9375651..d304601b9 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -50,11 +50,11 @@ class TypeDetector(provider: TypeProvider) {
       case Ast.expr.Bool(_) => CalcBooleanType
       case Ast.expr.EnumByLabel(enumType, _, inType) =>
         val t = EnumType(List(enumType.name), CalcIntType)
-        t.enumSpec = Some(provider.resolveEnum(enumType.name))
+        t.enumSpec = Some(provider.resolveEnum(inType, enumType.name))
         t
-      case Ast.expr.EnumById(enumType, _) =>
+      case Ast.expr.EnumById(enumType, _, inType) =>
         val t = EnumType(List(enumType.name), CalcIntType)
-        t.enumSpec = Some(provider.resolveEnum(enumType.name))
+        t.enumSpec = Some(provider.resolveEnum(inType, enumType.name))
         t
       case Ast.expr.Name(name: Ast.identifier) => provider.determineType(name.name)
       case Ast.expr.UnaryOp(op: Ast.unaryop, v: Ast.expr) =>
@@ -278,9 +278,11 @@ object TypeDetector {
       case (_: NumericType, _: NumericType) => // ok
       case (_: BooleanType, _: BooleanType) => // ok
       case (_: BytesType, _: BytesType) => // ok
-      case (EnumType(name1, _), EnumType(name2, _)) =>
-        if (name1 != name2) {
-          throw new TypeMismatchError(s"can't compare different enums '$name1' and '$name2'")
+      case (et1: EnumType, et2: EnumType) =>
+        val et1Spec = et1.enumSpec.get
+        val et2Spec = et2.enumSpec.get
+        if (et1Spec != et2Spec) {
+          throw new TypeMismatchError(s"can't compare different enums '${et1Spec.nameAsStr}' and '${et2Spec.nameAsStr}'")
         }
         op match {
           case Ast.cmpop.Eq | Ast.cmpop.NotEq => // ok
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala
index 119c33cc0..1964575ce 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeProvider.scala
@@ -13,7 +13,7 @@ trait TypeProvider {
   def nowClass: ClassSpec
   def determineType(attrName: String): DataType
   def determineType(inClass: ClassSpec, attrName: String): DataType
-  def resolveEnum(enumName: String): EnumSpec
+  def resolveEnum(typeName: Ast.typeId, enumName: String): EnumSpec
   def resolveType(typeName: Ast.typeId): DataType
   def isLazy(attrName: String): Boolean
   def isLazy(inClass: ClassSpec, attrName: String): Boolean

From dc18ec5a7e55154954faa418872f458ded7668a1 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Tue, 24 Apr 2018 20:13:14 +0100
Subject: [PATCH 147/230] Fixed some string formatting problems, string
 operation issues, and an import bug.

---
 .../io/kaitai/struct/languages/RustCompiler.scala      | 10 +++++-----
 .../io/kaitai/struct/translators/RustTranslator.scala  |  8 ++++----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index f50e98851..fc5d743da 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -173,7 +173,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def attrFixedContentsParse(attrName: Identifier, contents: String): Unit =
-    out.puts(s"${privateMemberName(attrName)} = $normalIO->ensureFixedContents($contents);")
+    out.puts(s"${privateMemberName(attrName)} = $normalIO.ensureFixedContents($contents);")
 
   override def attrProcess(proc: ProcessExpr, varSrc: Identifier, varDest: Identifier): Unit = {
     val srcName = privateMemberName(varSrc)
@@ -202,13 +202,13 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           importList.add(s"$onlyName::$className")
           className
         } else {
-          val pkgName = name.init.mkString(".")
+          val pkgName = type2classAbs(name.init)
           importList.add(s"$pkgName")
-          s"$pkgName.${type2class(name.last)}"
+          s"$pkgName::${type2class(name.last)}"
         }
 
-        out.puts(s"let _process = $procClass::new(${args.map(expression).mkString(", ")})")
-        out.puts(s"$destName = _process.decode($srcName)")
+        out.puts(s"let _process = $procClass::new(${args.map(expression).mkString(", ")});")
+        out.puts(s"$destName = _process.decode($srcName);")
     }
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
index ff17590f7..267ea06b4 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RustTranslator.scala
@@ -10,7 +10,7 @@ import io.kaitai.struct.{RuntimeConfig, Utils}
 class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends BaseTranslator(provider) {
   override def doByteArrayLiteral(arr: Seq[Byte]): String =
     "vec!([" + arr.map((x) =>
-    	     "0x%0.2X".format(x & 0xff)
+    	     "%0#2x".format(x & 0xff)
     ).mkString(", ") + "])"
   override def doByteArrayNonLiteral(elts: Seq[Ast.expr]): String =
     s"pack('C*', ${elts.map(translate).mkString(", ")})"
@@ -102,11 +102,11 @@ class RustTranslator(provider: TypeProvider, config: RuntimeConfig) extends Base
 	translate(encoding) + ")"
     }
   override def bytesLength(b: Ast.expr): String =
-    s"strlen(${translate(b)})"
+    s"${translate(b)}.len()"
   override def strLength(s: expr): String =
-    s"strlen(${translate(s)})"
+    s"${translate(s)}.len()"
   override def strReverse(s: expr): String =
-    s"strrev(${translate(s)})"
+    s"${translate(s)}.graphemes(true).rev().flat_map(|g| g.chars()).collect()"
   override def strSubstring(s: expr, from: expr, to: expr): String =
     s"${translate(s)}.substring(${translate(from)}, ${translate(to)})"
 

From 564aa4b5616a16a6c1598f622655b371b4393ddf Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 29 Apr 2018 14:11:00 +0100
Subject: [PATCH 148/230] RubyTranslator: fix deep enums resolution

---
 .../struct/translators/RubyTranslator.scala      | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
index 9a92be10c..686725f71 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -1,5 +1,6 @@
 package io.kaitai.struct.translators
 
+import io.kaitai.struct.Utils
 import io.kaitai.struct.datatype.DataType.EnumType
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.exprlang.Ast.expr
@@ -39,7 +40,20 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
   override def doEnumByLabel(enumTypeAbs: List[String], label: String): String =
     s":${enumTypeAbs.last}_$label"
   override def doEnumById(enumType: List[String], id: String): String =
-    s"${RubyCompiler.kstreamName}::resolve_enum(${enumType.last.toUpperCase}, $id)"
+    s"${RubyCompiler.kstreamName}::resolve_enum(${enumDirectMap(enumType)}, $id)"
+
+  def enumDirectMap(enumTypeAndName: List[String]): String = {
+    val enumTypeAbs = enumTypeAndName.dropRight(1)
+    val enumTypeName = enumTypeAndName.last.toUpperCase
+
+    val enumTypeRel = Utils.relClass(enumTypeAbs, provider.nowClass.name)
+
+    if (enumTypeRel.nonEmpty) {
+      (enumTypeRel.map((x) => Utils.upperCamelCase(x)) ++ List(enumTypeName)).mkString("::")
+    } else {
+      enumTypeName
+    }
+  }
 
   override def doSubscript(container: Ast.expr, idx: Ast.expr): String =
     s"${translate(container)}[${translate(idx)}]"

From e35d192980aef866e4c1d9ea097548e25f4ae496 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 8 Jul 2018 02:43:58 +0100
Subject: [PATCH 149/230] CppTranslator: fix doEnumByLabel to always generate a
 class hierarchy namespace

---
 .../scala/io/kaitai/struct/translators/CppTranslator.scala     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 130f7e967..891bb53cc 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -127,7 +127,8 @@ class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends B
   }
 
   override def doEnumByLabel(enumType: List[String], label: String): String =
-    (enumType.last + "_" + label).toUpperCase
+    CppCompiler.types2class(enumType.dropRight(1)) + "::" +
+      (enumType.last + "_" + label).toUpperCase
   override def doEnumById(enumType: List[String], id: String): String =
     s"static_cast<${CppCompiler.types2class(enumType)}>($id)"
 

From 6ef0e297f1fdab0f1758fbbf151846a7dcd2d826 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 8 Jul 2018 12:30:29 +0100
Subject: [PATCH 150/230] GoTranslator: do enum translation + comparisons

---
 .../kaitai/struct/translators/GoTranslator.scala   | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index 50312756a..c650cf9ca 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -71,11 +71,17 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
             trBytesCompareOp(left, op, right)
           case (_: BooleanType, _: BooleanType) =>
             trNumericCompareOp(left, op, right)
+          case (_: EnumType, _: EnumType) =>
+            trNumericCompareOp(left, op, right)
           case (ltype, rtype) =>
             throw new TypeMismatchError(s"can't do $ltype $op $rtype")
         }
-//      case Ast.expr.EnumByLabel(enumName, label) =>
-//      case Ast.expr.EnumById(enumName, id) =>
+      case Ast.expr.EnumById(enumType, id, inType) =>
+        val enumSpec = provider.resolveEnum(inType, enumType.name)
+        trEnumById(enumSpec.name, translate(id))
+      case Ast.expr.EnumByLabel(enumType, label, inType) =>
+        val enumSpec = provider.resolveEnum(inType, enumType.name)
+        trEnumByLabel(enumSpec.name, label.name)
       case ctt: Ast.expr.CastToType =>
         doCastOrArray(ctt)
       case Ast.expr.Subscript(container, idx) =>
@@ -207,8 +213,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     ResultLocalVar(v1)
   }
 
-//  override def doEnumByLabel(enumTypeAbs: List[String], label: String): String =
-//    s"${enumClass(enumTypeAbs)}.${label.toUpperCase}"
+  def trEnumByLabel(enumTypeAbs: List[String], label: String) =
+    ResultString(GoCompiler.enumToStr(enumTypeAbs, label))
   def trEnumById(enumTypeAbs: List[String], id: String) =
     ResultString(s"${types2class(enumTypeAbs)}($id)")
 

From c36a4b18f271905a668385016b55962520ed999e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 8 Jul 2018 16:05:58 +0100
Subject: [PATCH 151/230] GoTranslator: do enumToInt

---
 .../main/scala/io/kaitai/struct/translators/GoTranslator.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index c650cf9ca..c84d6178d 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -384,7 +384,8 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     ResultLocalVar(max)
   }
 
-  override def enumToInt(value: Ast.expr, et: EnumType): TranslatorResult = ???
+  override def enumToInt(value: Ast.expr, et: EnumType) =
+    translateExpr(value)
 
   override def boolToInt(value: Ast.expr): ResultLocalVar = {
     val v = allocateLocalVar()

From 4de87885a81d164c3f7ccc34381464ecc99813bc Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Mon, 16 Jul 2018 15:47:30 +0100
Subject: [PATCH 152/230] Fixed comparison with vector in Rust, when it occurs
 in switchCmp.

Rust match statements do not work with vector comparison, thus if
statements must be used instead.
---
 .../struct/languages/RustCompiler.scala       | 64 ++++++++++++++++---
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index fc5d743da..85f508df8 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -351,26 +351,74 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     expr2
   }
 
+  var switchIfs = false
+  val NAME_SWITCH_ON = Ast.expr.Name(Ast.identifier(Identifier.SWITCH_ON))
+
   override def switchStart(id: Identifier, on: Ast.expr): Unit = {
     val onType = translator.detectType(on)
 
-    out.puts(s"match ${expression(on)} {")
-    out.inc
+    switchIfs = onType match {
+      case _: ArrayType | _: BytesType => true
+      case _ => false
+    }
+
+    if (!switchIfs) {
+      out.puts(s"match ${expression(on)} {")
+      out.inc
+    }
+  }
+
+  def switchCmpExpr(condition: Ast.expr): String =
+    expression(
+      Ast.expr.Compare(
+        NAME_SWITCH_ON,
+	Ast.cmpop.Eq,
+	condition
+      )
+    )
+
+  override def switchCaseFirstStart(condition: Ast.expr): Unit = {
+    if (switchIfs) {
+      out.puts(s"if ${switchCmpExpr(condition)} {")
+      out.inc
+    } else {
+      switchCaseStart(condition)
+    }
   }
 
   override def switchCaseStart(condition: Ast.expr): Unit = {
-    out.puts(s"${expression(condition)} => {")
-    out.inc
+    if (switchIfs) {
+      out.puts(s"elss if ${switchCmpExpr(condition)} {")
+      out.inc
+    } else {
+      out.puts(s"${expression(condition)} => {")
+      out.inc
+    }
   }
 
   override def switchCaseEnd(): Unit = {
-    out.dec
-    out.puts("},")
+    if (switchIfs) {
+      out.dec
+      out.puts("}")
+    } else {
+      out.dec
+      out.puts("},")
+    }
   }
 
   override def switchElseStart(): Unit = {
-    out.puts("_ => {")
-    out.inc
+    if (switchIfs) {
+      out.puts("else {")
+      out.inc
+    } else {
+      out.puts("_ => {")
+      out.inc
+    }
+  }
+
+  override def switchElseEnd(): Unit = {
+    out.dec
+    out.puts("}")
   }
 
   override def switchEnd(): Unit = universalFooter

From 18cecf49a36446e6d0e2e3d98a00f1c346e7446a Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 16 Jul 2018 17:32:27 +0100
Subject: [PATCH 153/230] Added --debug support for Python

---
 .../struct/languages/PythonCompiler.scala     | 39 +++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index 6026981fe..b320636a2 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -94,6 +94,11 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
     // Store parameters passed to us
     params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id)))
+
+    if (debug) {
+      importList.add("import collections")
+      out.puts("self._debug = collections.defaultdict(dict)")
+    }
   }
 
   override def runRead(): Unit = {
@@ -248,6 +253,40 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def alignToByte(io: String): Unit =
     out.puts(s"$io.align_to_byte()")
 
+  override def attrDebugStart(attrId: Identifier, attrType: DataType, ios: Option[String], rep: RepeatSpec): Unit = {
+    ios.foreach { (io) =>
+      val name = attrId match {
+        case _: RawIdentifier | _: SpecialIdentifier => return
+        case _ => idToStr(attrId)
+      }
+      rep match {
+        case NoRepeat =>
+          out.puts(s"self._debug['$name']['start'] = $io.pos()")
+        case _: RepeatExpr | RepeatEos | _: RepeatUntil =>
+          out.puts(s"if not 'arr' in self._debug['$name']:")
+          out.inc
+          out.puts(s"self._debug['$name']['arr'] = []")
+          out.dec
+          out.puts(s"self._debug['$name']['arr'].append({'start': $io.pos()})")
+      }
+    }
+  }
+
+  override def attrDebugEnd(attrId: Identifier, attrType: DataType, io: String, rep: RepeatSpec): Unit = {
+    val name = attrId match {
+      case _: RawIdentifier | _: SpecialIdentifier => return
+      case _ => idToStr(attrId)
+    }
+    rep match {
+      case NoRepeat =>
+        out.puts(s"self._debug['$name']['end'] = $io.pos()")
+      case _: RepeatExpr =>
+        out.puts(s"self._debug['$name']['arr'][i]['end'] = $io.pos()")
+      case RepeatEos | _: RepeatUntil =>
+        out.puts(s"self._debug['$name']['arr'][len(${privateMemberName(attrId)}) - 1]['end'] = $io.pos()")
+    }
+  }
+
   override def condIfHeader(expr: Ast.expr): Unit = {
     out.puts(s"if ${expression(expr)}:")
     out.inc

From 23f884144b9b8f4ad0913e61db6606b2791949a6 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 4 Aug 2018 16:12:01 +0100
Subject: [PATCH 154/230] CppCompiler: moved kaitaiType2NativeType to object

---
 .../kaitai/struct/languages/CppCompiler.scala | 110 +++++++++---------
 1 file changed, 55 insertions(+), 55 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 1e85b57e9..2278ad1d1 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -752,61 +752,6 @@ class CppCompiler(
 
   def value2Const(enumName: String, label: String) = (enumName + "_" + label).toUpperCase
 
-  def kaitaiType2NativeType(attrType: DataType, absolute: Boolean = false): String = {
-    attrType match {
-      case Int1Type(false) => "uint8_t"
-      case IntMultiType(false, Width2, _) => "uint16_t"
-      case IntMultiType(false, Width4, _) => "uint32_t"
-      case IntMultiType(false, Width8, _) => "uint64_t"
-
-      case Int1Type(true) => "int8_t"
-      case IntMultiType(true, Width2, _) => "int16_t"
-      case IntMultiType(true, Width4, _) => "int32_t"
-      case IntMultiType(true, Width8, _) => "int64_t"
-
-      case FloatMultiType(Width4, _) => "float"
-      case FloatMultiType(Width8, _) => "double"
-
-      case BitsType(_) => "uint64_t"
-
-      case _: BooleanType => "bool"
-      case CalcIntType => "int32_t"
-      case CalcFloatType => "double"
-
-      case _: StrType => "std::string"
-      case _: BytesType => "std::string"
-
-      case t: UserType =>
-        val typeStr = types2class(if (absolute) {
-          t.classSpec.get.name
-        } else {
-          t.name
-        })
-        s"$typeStr*"
-
-      case t: EnumType =>
-        types2class(if (absolute) {
-          t.enumSpec.get.name
-        } else {
-          t.name
-        })
-
-      case ArrayType(inType) => s"std::vector<${kaitaiType2NativeType(inType, absolute)}>*"
-
-      case KaitaiStreamType => s"$kstreamName*"
-      case KaitaiStructType => s"$kstructName*"
-
-      case SwitchType(on, cases) =>
-        kaitaiType2NativeType(TypeDetector.combineTypes(
-          // C++ does not have a concept of AnyType, and common use case "lots of incompatible UserTypes
-          // for cases + 1 BytesType for else" combined would result in exactly AnyType - so we try extra
-          // hard to avoid that here with this pre-filtering. In C++, "else" case with raw byte array would
-          // be available through _raw_* attribute anyway.
-          cases.filterNot { case (caseExpr, caseValue) => caseExpr == SwitchType.ELSE_CONST }.values
-        ), absolute)
-    }
-  }
-
   def defineName(className: String) = className.toUpperCase + "_H_"
 
   /**
@@ -869,6 +814,61 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
   override def kstructName = "kaitai::kstruct"
   override def kstreamName = "kaitai::kstream"
 
+  def kaitaiType2NativeType(attrType: DataType, absolute: Boolean = false): String = {
+    attrType match {
+      case Int1Type(false) => "uint8_t"
+      case IntMultiType(false, Width2, _) => "uint16_t"
+      case IntMultiType(false, Width4, _) => "uint32_t"
+      case IntMultiType(false, Width8, _) => "uint64_t"
+
+      case Int1Type(true) => "int8_t"
+      case IntMultiType(true, Width2, _) => "int16_t"
+      case IntMultiType(true, Width4, _) => "int32_t"
+      case IntMultiType(true, Width8, _) => "int64_t"
+
+      case FloatMultiType(Width4, _) => "float"
+      case FloatMultiType(Width8, _) => "double"
+
+      case BitsType(_) => "uint64_t"
+
+      case _: BooleanType => "bool"
+      case CalcIntType => "int32_t"
+      case CalcFloatType => "double"
+
+      case _: StrType => "std::string"
+      case _: BytesType => "std::string"
+
+      case t: UserType =>
+        val typeStr = types2class(if (absolute) {
+          t.classSpec.get.name
+        } else {
+          t.name
+        })
+        s"$typeStr*"
+
+      case t: EnumType =>
+        types2class(if (absolute) {
+          t.enumSpec.get.name
+        } else {
+          t.name
+        })
+
+      case ArrayType(inType) => s"std::vector<${kaitaiType2NativeType(inType, absolute)}>*"
+
+      case KaitaiStreamType => s"$kstreamName*"
+      case KaitaiStructType => s"$kstructName*"
+
+      case SwitchType(on, cases) =>
+        kaitaiType2NativeType(TypeDetector.combineTypes(
+          // C++ does not have a concept of AnyType, and common use case "lots of incompatible UserTypes
+          // for cases + 1 BytesType for else" combined would result in exactly AnyType - so we try extra
+          // hard to avoid that here with this pre-filtering. In C++, "else" case with raw byte array would
+          // be available through _raw_* attribute anyway.
+          cases.filterNot { case (caseExpr, caseValue) => caseExpr == SwitchType.ELSE_CONST }.values
+        ), absolute)
+    }
+  }
+
   def types2class(typeName: Ast.typeId) = {
     typeName.names.map(type2class).mkString(
       if (typeName.absolute) "::" else "",

From bf4edfe30a0b77c6858c70d87c9add742129483e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 4 Aug 2018 16:13:33 +0100
Subject: [PATCH 155/230] Allow pure types in as<...> casting: added tests +
 implementation fixes for C# / C++ / Java

---
 .../kaitai/struct/translators/TranslatorSpec.scala | 14 ++++++++++++++
 .../kaitai/struct/translators/BaseTranslator.scala |  2 +-
 .../struct/translators/CSharpTranslator.scala      |  4 ++--
 .../struct/translators/CommonArraysAndCast.scala   | 10 ++++++----
 .../kaitai/struct/translators/CppTranslator.scala  |  4 ++--
 .../kaitai/struct/translators/GoTranslator.scala   |  2 +-
 .../kaitai/struct/translators/JavaTranslator.scala |  6 ++----
 7 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index f8000d53a..ff898d0a0 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -488,6 +488,20 @@ class TranslatorSpec extends FunSuite {
     RubyCompiler -> "other.baz"
   ))
 
+  // primitive pure types
+  full("(1 + 2).as<s2>", CalcIntType, IntMultiType(true, Width2, None), Map[LanguageCompilerStatic, String](
+    CppCompiler -> "static_cast<int16_t>((1 + 2))",
+    CSharpCompiler -> "((short) ((1 + 2)))",
+    GoCompiler -> "int16((1 + 2))",
+    JavaCompiler -> "((short) ((1 + 2)))",
+    JavaScriptCompiler -> "(1 + 2)",
+    LuaCompiler -> "(1 + 2)",
+    PerlCompiler -> "(1 + 2)",
+    PHPCompiler -> "(1 + 2)",
+    PythonCompiler -> "(1 + 2)",
+    RubyCompiler -> "(1 + 2)"
+  ))
+
   // empty array casting
   full("[].as<bytes>", CalcIntType, CalcBytesType, Map[LanguageCompilerStatic, String](
     CppCompiler -> "std::string(\"\", 0)",
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
index 6aca17e93..9d8dedbd0 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/BaseTranslator.scala
@@ -129,7 +129,7 @@ abstract class BaseTranslator(val provider: TypeProvider)
 
   def doSubscript(container: Ast.expr, idx: Ast.expr): String
   def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String
-  def doCast(value: Ast.expr, typeName: Ast.typeId): String = translate(value)
+  def doCast(value: Ast.expr, typeName: DataType): String = translate(value)
 
   def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): String = "[" + value.map((v) => translate(v)).mkString(", ") + "]"
   def doByteArrayLiteral(arr: Seq[Byte]): String = "[" + arr.map(_ & 0xff).mkString(", ") + "]"
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
index fea6fc5bc..31c948e1a 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CSharpTranslator.scala
@@ -83,8 +83,8 @@ class CSharpTranslator(provider: TypeProvider, importList: ImportList) extends B
     s"${translate(container)}[${translate(idx)}]"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
-  override def doCast(value: Ast.expr, typeName: Ast.typeId): String =
-    s"((${CSharpCompiler.types2class(typeName)}) (${translate(value)}))"
+  override def doCast(value: Ast.expr, typeName: DataType): String =
+    s"((${CSharpCompiler.kaitaiType2NativeType(typeName)}) (${translate(value)}))"
 
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String = {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala b/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala
index 900c7b23f..945ec52b8 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CommonArraysAndCast.scala
@@ -51,11 +51,13 @@ trait CommonArraysAndCast[T] extends TypeDetector {
     * @return translation result
     */
   def doCastOrArray(v: Ast.expr.CastToType): T = {
+    val castToType = detectCastType(v.typeName)
+
     v.value match {
       case array: Ast.expr.List =>
         // Special handling for literal arrays: if cast is present,
         // then we don't need to guess the data type
-        detectCastType(v.typeName) match {
+        castToType match {
           case _: BytesType =>
             doByteArray(array.elts)
           case ArrayType(elType) =>
@@ -63,14 +65,14 @@ trait CommonArraysAndCast[T] extends TypeDetector {
           case _ =>
             // No luck, this is some kind of weird cast, not a type enforcement;
             // Just do it and let real type casting deal with it.
-            doCast(v.value, v.typeName)
+            doCast(v.value, castToType)
         }
       case _ =>
-        doCast(v.value, v.typeName)
+        doCast(v.value, castToType)
     }
   }
 
-  def doCast(value: Ast.expr, typeName: Ast.typeId): T
+  def doCast(value: Ast.expr, typeName: DataType): T
   def doArrayLiteral(t: DataType, value: Seq[Ast.expr]): T
   def doByteArrayLiteral(arr: Seq[Byte]): T
   def doByteArrayNonLiteral(elts: Seq[Ast.expr]): T
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 891bb53cc..f7eb61606 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -146,8 +146,8 @@ class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends B
     s"${translate(container)}->at(${translate(idx)})"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"((${translate(condition)}) ? (${translate(ifTrue)}) : (${translate(ifFalse)}))"
-  override def doCast(value: Ast.expr, typeName: Ast.typeId): String =
-    s"static_cast<${CppCompiler.types2class(typeName)}*>(${translate(value)})"
+  override def doCast(value: Ast.expr, typeName: DataType): String =
+    s"static_cast<${CppCompiler.kaitaiType2NativeType(typeName)}>(${translate(value)})"
 
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String = {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
index c84d6178d..358c1e3b1 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/GoTranslator.scala
@@ -229,7 +229,7 @@ class GoTranslator(out: StringLanguageOutputWriter, provider: TypeProvider, impo
     }
   }
 
-  override def doCast(value: Ast.expr, typeName: Ast.typeId): TranslatorResult = ???
+  override def doCast(value: Ast.expr, typeName: DataType): TranslatorResult = ???
 
   override def doArrayLiteral(t: DataType, value: Seq[Ast.expr]) =
     ResultString(s"[]${GoCompiler.kaitaiType2NativeType(t)}{${value.map(translate).mkString(", ")}}")
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
index a524a25f5..1c2320f87 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/JavaTranslator.scala
@@ -92,10 +92,8 @@ class JavaTranslator(provider: TypeProvider, importList: ImportList) extends Bas
     s"${translate(container)}.get((int) ${translate(idx)})"
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"(${translate(condition)} ? ${translate(ifTrue)} : ${translate(ifFalse)})"
-  override def doCast(value: Ast.expr, typeName: Ast.typeId): String = {
-    // FIXME: assuming relative type name
-    s"((${JavaCompiler.types2class(typeName.names.toList)}) (${translate(value)}))"
-  }
+  override def doCast(value: Ast.expr, typeName: DataType): String =
+    s"((${JavaCompiler.kaitaiType2JavaType(typeName)}) (${translate(value)}))"
 
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String =

From e94bbaf96048806ad3dd726bb55e0e99da57f646 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 6 Aug 2018 11:53:31 +0100
Subject: [PATCH 156/230] PythonCompiler: fixed missing handleAssignmentTempVar

---
 .../main/scala/io/kaitai/struct/languages/PythonCompiler.scala | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index b320636a2..b2591f7a4 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -345,6 +345,9 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def handleAssignmentSimple(id: Identifier, expr: String): Unit =
     out.puts(s"${privateMemberName(id)} = $expr")
 
+  override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit =
+    out.puts(s"$id = $expr")
+
   override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = {
     dataType match {
       case t: ReadableType =>

From 41e9a067d78d8f2855f47a4f44bcd2874b2e3c88 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 7 Aug 2018 18:41:49 +0100
Subject: [PATCH 157/230] ParamsDefSpec: enforce legal keys check + support
 enums

---
 .../main/scala/io/kaitai/struct/datatype/DataType.scala  | 9 ++++++++-
 .../scala/io/kaitai/struct/format/ParamDefSpec.scala     | 7 ++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index 0a7bd677e..56be8ce07 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -301,7 +301,11 @@ object DataType {
       }
     }
 
-    arg.enumRef match {
+    applyEnumType(r, arg.enumRef, path)
+  }
+
+  private def applyEnumType(r: DataType, enumRef: Option[String], path: List[String]) = {
+    enumRef match {
       case Some(enumName) =>
         r match {
           case numType: IntType => EnumType(classNameToList(enumName), numType)
@@ -316,6 +320,9 @@ object DataType {
   private val RePureIntType = """([us])(2|4|8)""".r
   private val RePureFloatType = """f(4|8)""".r
 
+  def pureFromString(dto: Option[String], enumRef: Option[String], path: List[String]): DataType =
+    applyEnumType(pureFromString(dto), enumRef, path)
+
   def pureFromString(dto: Option[String]): DataType = dto match {
     case None => CalcBytesType
     case Some(dt) => pureFromString(dt)
diff --git a/shared/src/main/scala/io/kaitai/struct/format/ParamDefSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ParamDefSpec.scala
index b5cef37be..5d25eaf62 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/ParamDefSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/ParamDefSpec.scala
@@ -22,6 +22,7 @@ object ParamDefSpec {
   val LEGAL_KEYS = Set(
     "id",
     "type",
+    "enum",
     "doc",
     "doc-ref"
   )
@@ -29,7 +30,11 @@ object ParamDefSpec {
   def fromYaml(srcMap: Map[String, Any], path: List[String], id: Identifier): ParamDefSpec = {
     val doc = DocSpec.fromYaml(srcMap, path)
     val typeStr = ParseUtils.getOptValueStr(srcMap, "type", path)
-    val dataType = DataType.pureFromString(typeStr)
+    val enumRef = ParseUtils.getOptValueStr(srcMap, "enum", path)
+
+    val dataType = DataType.pureFromString(typeStr, enumRef, path)
+
+    ParseUtils.ensureLegalKeys(srcMap, LEGAL_KEYS, path, Some("parameter definition"))
 
     ParamDefSpec(path, id, dataType, doc)
   }

From da2caedb61e6ce353ed1c245665372fdd7b1911c Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Thu, 9 Aug 2018 22:28:58 +0100
Subject: [PATCH 158/230] Fixed stuff

---
 .../scala/io/kaitai/struct/languages/RustCompiler.scala    | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index 85f508df8..b43908a24 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -203,8 +203,9 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           className
         } else {
           val pkgName = type2classAbs(name.init)
-          importList.add(s"$pkgName")
-          s"$pkgName::${type2class(name.last)}"
+	  val className = type2class(name.last)
+          importList.add(s"$pkgName::$className")
+          s"$pkgName::$className"
         }
 
         out.puts(s"let _process = $procClass::new(${args.map(expression).mkString(", ")});")
@@ -216,7 +217,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     val memberName = privateMemberName(id)
 
     val args = rep match {
-      case RepeatEos | RepeatExpr(_) => s"end($memberName)"
+      case RepeatEos | RepeatExpr(_) => s"$memberName.last()"
       case RepeatUntil(_) => translator.doLocalName(Identifier.ITERATOR2)
       case NoRepeat => memberName
     }

From a4b5d2192757103795793cdcdc4f5dc4e3edf255 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 15 Aug 2018 21:02:22 +0100
Subject: [PATCH 159/230] ExtraAttrs: respect array in ConditionSpec to
 generate extra members

---
 .../kaitai/struct/languages/components/ExtraAttrs.scala  | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
index a1f9e6f5c..75f2ba6bd 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
@@ -14,20 +14,21 @@ import io.kaitai.struct.format._
   */
 object ExtraAttrs {
   def forAttr(attr: AttrLikeSpec): List[AttrSpec] =
-    forAttr(attr.id, attr.dataType)
+    forAttr(attr.id, attr.dataType, attr.cond)
 
-  def forAttr(id: Identifier, dataType: DataType): List[AttrSpec] = {
+  private
+  def forAttr(id: Identifier, dataType: DataType, condSpec: ConditionalSpec): List[AttrSpec] = {
     dataType match {
       case bt: BytesType =>
         bt.process match {
           case None => List()
           case Some(_) =>
             val rawId = RawIdentifier(id)
-            List(AttrSpec(List(), rawId, bt))
+            List(AttrSpec(List(), rawId, bt, condSpec))
         }
       case utb: UserTypeFromBytes =>
         val rawId = RawIdentifier(id)
-        List(AttrSpec(List(), rawId, utb.bytes)) ++ forAttr(rawId, utb.bytes)
+        List(AttrSpec(List(), rawId, utb.bytes, condSpec)) ++ forAttr(rawId, utb.bytes, condSpec)
       case _ =>
         List()
     }

From 5c240f96a2bbb7e8f362801f6576138db8ce2a60 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 15 Aug 2018 21:03:03 +0100
Subject: [PATCH 160/230] GoCompiler: fix initialization of array-based extra
 members

---
 .../scala/io/kaitai/struct/languages/GoCompiler.scala     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index 9421e2b59..c3a4e7c50 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -160,7 +160,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     val ioName = idToStr(IoStorageIdentifier(varName))
 
     val args = rep match {
-      case RepeatEos | RepeatExpr(_) => s"$javaName.get($javaName.size() - 1)"
+      case RepeatEos | RepeatExpr(_) => s"$javaName[len($javaName) - 1]"
       case RepeatUntil(_) => translator.specialName(Identifier.ITERATOR2)
       case NoRepeat => javaName
     }
@@ -205,7 +205,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatEosHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>();")
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = make([][]byte, 0);")
     //out.puts(s"${privateMemberName(id)} = make(${kaitaiType2NativeType(ArrayType(dataType))})")
     out.puts(s"for {")
     out.inc
@@ -228,7 +228,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: Ast.expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>((int) (${expression(repeatExpr)}));")
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = make([][]byte, ${expression(repeatExpr)})")
     out.puts(s"${privateMemberName(id)} = make(${kaitaiType2NativeType(ArrayType(dataType))}, ${expression(repeatExpr)})")
     out.puts(s"for i := range ${privateMemberName(id)} {")
     out.inc
@@ -242,7 +242,7 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatUntilHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: Ast.expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>();")
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = make([][]byte, 0);")
     out.puts("for {")
     out.inc
   }

From bc59043379ffbb9269f17edd09b3d79cf53842e0 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 15 Aug 2018 21:03:57 +0100
Subject: [PATCH 161/230] GoClassCompiler: preparing to get rid of extraAttrs
 where possible; where it's not possible, replaced with clearly dummy empty
 ExtraAttrs

---
 .../src/main/scala/io/kaitai/struct/GoClassCompiler.scala | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
index 733dc1577..5857f944a 100644
--- a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
@@ -39,7 +39,7 @@ class GoClassCompiler(
     lang.classFooter(curClass.name)
 
     // Constructor = Read() function
-    compileReadFunction(curClass, extraAttrs)
+    compileReadFunction(curClass)
 
     compileInstances(curClass, extraAttrs)
 
@@ -49,7 +49,7 @@ class GoClassCompiler(
     compileSubclasses(curClass)
   }
 
-  def compileReadFunction(curClass: ClassSpec, extraAttrs: ListBuffer[AttrSpec]) = {
+  def compileReadFunction(curClass: ClassSpec) = {
     lang.classConstructorHeader(
       curClass.name,
       curClass.parentType,
@@ -62,7 +62,7 @@ class GoClassCompiler(
       case Some(fe: FixedEndian) => Some(fe)
       case _ => None
     }
-    compileSeq(curClass.seq, extraAttrs, defEndian)
+    compileSeq(curClass.seq, new ListBuffer[AttrSpec], defEndian)
     lang.classConstructorFooter
   }
 
@@ -92,7 +92,7 @@ class GoClassCompiler(
   }
 
   def getExtraAttrs(curClass: ClassSpec): List[AttrSpec] = {
-    curClass.seq.foldLeft(List[AttrSpec]())(
+    (curClass.seq ++ curClass.instances.values.asInstanceOf[Iterable[AttrLikeSpec]]).foldLeft(List[AttrSpec]())(
       (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr)
     )
   }

From 5ce073e11ad02f4510900d25c7a1cb0ef7c17275 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 18 Aug 2018 12:14:23 +0100
Subject: [PATCH 162/230] GoClassCompiler: fixed bug with invalid cast for
 ValueInstances

---
 .../main/scala/io/kaitai/struct/GoClassCompiler.scala    | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
index 5857f944a..bb3e9a0db 100644
--- a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
@@ -92,7 +92,14 @@ class GoClassCompiler(
   }
 
   def getExtraAttrs(curClass: ClassSpec): List[AttrSpec] = {
-    (curClass.seq ++ curClass.instances.values.asInstanceOf[Iterable[AttrLikeSpec]]).foldLeft(List[AttrSpec]())(
+    // We want only values of ParseInstances, which are AttrSpecLike.
+    // ValueInstances are ignored, as they can't currently generate
+    // any extra attributes (i.e. no `size`, no `process`, etc)
+    val parseInstances = curClass.instances.values.collect {
+      case inst: AttrLikeSpec => inst
+    }
+
+    (curClass.seq ++ parseInstances).foldLeft(List[AttrSpec]())(
       (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr)
     )
   }

From 178aef7357cd005be3dd84ef10553912e78d7f91 Mon Sep 17 00:00:00 2001
From: Connor Wood <connorwood71@gmail.com>
Date: Sat, 18 Aug 2018 12:36:51 +0100
Subject: [PATCH 163/230] Fixed linter bugs regarding imports, trailing spaces

---
 .../main/scala/io/kaitai/struct/languages/RustCompiler.scala  | 4 +---
 .../kaitai/struct/languages/components/LanguageCompiler.scala | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index b43908a24..fc9965300 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -1,14 +1,12 @@
 package io.kaitai.struct.languages
 
-import io.kaitai.struct._
+import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils, _}
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian}
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.{NoRepeat, RepeatEos, RepeatExpr, RepeatSpec, _}
-import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
 import io.kaitai.struct.translators.{RustTranslator, TypeDetector}
-import io.kaitai.struct.{ClassTypeProvider, RuntimeConfig, Utils}
 
 class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   extends LanguageCompiler(typeProvider, config)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index e1392a4cb..f4929ff5c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -116,7 +116,7 @@ abstract class LanguageCompiler(
   def seek(io: String, pos: Ast.expr): Unit
   def popPos(io: String): Unit
   def alignToByte(io: String): Unit
-  
+
   def instanceDeclHeader(className: List[String]): Unit = {}
   def instanceClear(instName: InstanceIdentifier): Unit = {}
   def instanceSetCalculated(instName: InstanceIdentifier): Unit = {}

From 2804e74f8519e6d7eadda864e11e47caec121615 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 2 Sep 2018 03:01:37 +0100
Subject: [PATCH 164/230] Clearly separated --debug into --no-auto-read and
 --read-pos

---
 .../main/scala/io/kaitai/struct/MainJs.scala  |  2 +-
 .../scala/io/kaitai/struct/JavaMain.scala     | 13 +++++++--
 .../io/kaitai/struct/ClassCompiler.scala      |  4 +--
 .../main/scala/io/kaitai/struct/Main.scala    |  2 +-
 .../io/kaitai/struct/RuntimeConfig.scala      | 27 ++++++++++++++++++-
 .../struct/languages/CSharpCompiler.scala     |  2 +-
 .../kaitai/struct/languages/CppCompiler.scala |  4 ++-
 .../struct/languages/JavaCompiler.scala       |  4 +--
 .../struct/languages/JavaScriptCompiler.scala |  8 +++---
 .../struct/languages/PythonCompiler.scala     |  2 +-
 .../struct/languages/RubyCompiler.scala       |  4 +--
 .../languages/components/CommonReads.scala    |  4 +--
 .../components/EveryReadIsExpression.scala    | 16 +++++------
 .../components/LanguageCompiler.scala         |  4 +--
 14 files changed, 66 insertions(+), 30 deletions(-)

diff --git a/js/src/main/scala/io/kaitai/struct/MainJs.scala b/js/src/main/scala/io/kaitai/struct/MainJs.scala
index 2f16766e3..342054305 100644
--- a/js/src/main/scala/io/kaitai/struct/MainJs.scala
+++ b/js/src/main/scala/io/kaitai/struct/MainJs.scala
@@ -16,7 +16,7 @@ object MainJs {
   @JSExport
   def compile(langStr: String, yaml: js.Object, importer: JavaScriptImporter, debug: Boolean = false): js.Promise[js.Dictionary[String]] = {
     try {
-      val config = new RuntimeConfig(debug = debug)
+      val config = new RuntimeConfig(autoRead = !debug, readStoresPos = debug)
       val lang = LanguageCompilerStatic.byString(langStr)
 
       val yamlScala = JavaScriptKSYParser.yamlJavascriptToScala(yaml)
diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
index 712b3a549..97bc2fbe7 100644
--- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
+++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
@@ -115,9 +115,18 @@ object JavaMain {
         }
       }
 
+      opt[Unit]("no-auto-read") action { (x, c) =>
+        c.copy(runtime = c.runtime.copy(autoRead = false))
+      } text("disable auto-running `_read` in constructor")
+
+      opt[Unit]("read-pos") action { (x, c) =>
+        c.copy(runtime = c.runtime.copy(readStoresPos = true))
+      } text("`_read` remembers attribute positions in stream")
+
       opt[Unit]("debug") action { (x, c) =>
-        c.copy(runtime = c.runtime.copy(debug = true))
-      } text("enable debugging helpers (mostly used by visualization tools)")
+        c.copy(runtime = c.runtime.copy(autoRead = false, readStoresPos = true))
+      } text("same as --no-auto-read --read-pos (useful for visualization tools)")
+
       help("help") text("display this help and exit")
       version("version") text("output version information and exit")
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index 660ba8522..f30cd317a 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -61,7 +61,7 @@ class ClassCompiler(
     if (lang.innerEnums)
       compileEnums(curClass)
 
-    if (lang.debug)
+    if (lang.config.readStoresPos)
       lang.debugClassSequence(curClass.seq)
 
     // Constructor
@@ -114,7 +114,7 @@ class ClassCompiler(
       curClass.params
     )
     curClass.instances.foreach { case (instName, _) => lang.instanceClear(instName) }
-    if (!lang.debug)
+    if (lang.config.autoRead)
       lang.runRead()
     lang.classConstructorFooter
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala
index 299185072..08e0ca514 100644
--- a/shared/src/main/scala/io/kaitai/struct/Main.scala
+++ b/shared/src/main/scala/io/kaitai/struct/Main.scala
@@ -78,7 +78,7 @@ object Main {
     */
   private def updateConfig(config: RuntimeConfig, topClass: ClassSpec): RuntimeConfig = {
     if (topClass.meta.forceDebug) {
-      config.copy(debug = true)
+      config.copy(autoRead = false, readStoresPos = true)
     } else {
       config
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
index aa7a56abf..f31147725 100644
--- a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
@@ -1,7 +1,32 @@
 package io.kaitai.struct
 
+/**
+  * Runtime configuration of the compiler which controls certain aspects of
+  * code generation for target languages.
+  * @param autoRead If true, constructor (or equivalent) invocation would
+  *                 automatically run `_read` (or equivalent), thus allowing to
+  *                 run parsing just by constructing an object, passing a stream
+  *                 into it. If false, `_read` would be made public and it is
+  *                 expected to be invoked manually.
+  * @param readStoresPos If true, parser (`_read` or equivalent) will store
+  *                      positions of all the attributes relative to the stream;
+  *                      not required for production usage (as it is typically slow
+  *                      and memory-consuming), but it is crucial for visualizers,
+  *                      IDEs, etc, to be able to display data layout.
+  * @param opaqueTypes If true, invoking any unknown type will be treated as it was
+  *                    "opaque" type, i.e. an external KaitaiStruct-compatible type
+  *                    defined somewhere else. If false, it will be reported as
+  *                    precompile error.
+  * @param goPackage Go package name
+  * @param javaPackage Java package name
+  * @param javaFromFileClass Java class to be invoked in `fromFile` helper methods
+  * @param dotNetNamespace .NET (C#) namespace
+  * @param phpNamespace PHP namespace
+  * @param pythonPackage Python package name
+  */
 case class RuntimeConfig(
-  debug: Boolean = false,
+  autoRead: Boolean = true,
+  readStoresPos: Boolean = false,
   opaqueTypes: Boolean = false,
   goPackage: String = "",
   javaPackage: String = "",
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
index 50e0fa128..db80b797f 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
@@ -135,7 +135,7 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = {
-    val readAccessAndType = if (debug) {
+    val readAccessAndType = if (!config.autoRead) {
       "public"
     } else {
       "private"
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 2278ad1d1..c35ec6dca 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -216,7 +216,9 @@ class CppCompiler(
       case Some(e) => s"_${e.toSuffix}"
       case None => ""
     }
-    ensureMode(PrivateAccess)
+
+    ensureMode(if (config.autoRead) PrivateAccess else PublicAccess)
+
     outHdr.puts(s"void _read$suffix();")
     outSrc.puts
     outSrc.puts(s"void ${types2class(typeProvider.nowClass.name)}::_read$suffix() {")
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 25247b83a..4eee672e5 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -72,7 +72,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"public ${staticStr}class ${type2class(name)} extends $kstructName {")
     out.inc
 
-    if (debug) {
+    if (config.readStoresPos) {
       out.puts("public Map<String, Integer> _attrStart = new HashMap<String, Integer>();")
       out.puts("public Map<String, Integer> _attrEnd = new HashMap<String, Integer>();")
       out.puts("public Map<String, ArrayList<Integer>> _arrStart = new HashMap<String, ArrayList<Integer>>();")
@@ -179,7 +179,7 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def readHeader(endian: Option[FixedEndian], isEmpty: Boolean) = {
-    val readAccessAndType = if (debug) {
+    val readAccessAndType = if (!config.autoRead) {
       "public"
     } else {
       "private"
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
index 7361f27b2..a3842f748 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
@@ -103,7 +103,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     // Store parameters passed to us
     params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id)))
 
-    if (debug) {
+    if (config.readStoresPos) {
       out.puts("this._debug = {};")
     }
     out.puts
@@ -295,7 +295,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     if (needRaw)
       out.puts(s"${privateMemberName(RawIdentifier(id))} = [];")
     out.puts(s"${privateMemberName(id)} = [];")
-    if (debug)
+    if (config.readStoresPos)
       out.puts(s"this._debug.${idToStr(id)}.arr = [];")
     out.puts("var i = 0;")
     out.puts(s"while (!$io.isEof()) {")
@@ -316,7 +316,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     if (needRaw)
       out.puts(s"${privateMemberName(RawIdentifier(id))} = new Array(${expression(repeatExpr)});")
     out.puts(s"${privateMemberName(id)} = new Array(${expression(repeatExpr)});")
-    if (debug)
+    if (config.readStoresPos)
       out.puts(s"this._debug.${idToStr(id)}.arr = new Array(${expression(repeatExpr)});")
     out.puts(s"for (var i = 0; i < ${expression(repeatExpr)}; i++) {")
     out.inc
@@ -335,7 +335,7 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     if (needRaw)
       out.puts(s"${privateMemberName(RawIdentifier(id))} = []")
     out.puts(s"${privateMemberName(id)} = []")
-    if (debug)
+    if (config.readStoresPos)
       out.puts(s"this._debug.${idToStr(id)}.arr = [];")
     out.puts("var i = 0;")
     out.puts("do {")
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index b2591f7a4..3333f5e08 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -95,7 +95,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     // Store parameters passed to us
     params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id)))
 
-    if (debug) {
+    if (config.readStoresPos) {
       importList.add("import collections")
       out.puts("self._debug = collections.defaultdict(dict)")
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index 9138981e1..3788276bb 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -64,7 +64,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   override def classHeader(name: String): Unit = {
     out.puts(s"class ${type2class(name)} < $kstructName")
     out.inc
-    if (debug)
+    if (config.readStoresPos)
       out.puts("attr_reader :_debug")
   }
 
@@ -88,7 +88,7 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     // Store parameters passed to us
     params.foreach((p) => handleAssignmentSimple(p.id, paramName(p.id)))
 
-    if (debug) {
+    if (config.readStoresPos) {
       out.puts("@_debug = {}")
     }
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala
index 350d1c091..8bf4b7a5c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala
@@ -28,7 +28,7 @@ trait CommonReads extends LanguageCompiler {
         normalIO
     }
 
-    if (debug)
+    if (config.readStoresPos)
       attrDebugStart(id, attr.dataType, Some(io), NoRepeat)
 
     defEndian match {
@@ -43,7 +43,7 @@ trait CommonReads extends LanguageCompiler {
         attrParse0(id, attr, io, extraAttrs, Some(fe))
     }
 
-    if (debug)
+    if (config.readStoresPos)
       attrDebugEnd(id, attr.dataType, io, NoRepeat)
 
     // More position management after parsing for ParseInstanceSpecs
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
index fcee72026..41ae56128 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
@@ -33,7 +33,7 @@ trait EveryReadIsExpression
   ): Unit = {
     val assignType = assignTypeOpt.getOrElse(dataType)
 
-    if (debug && rep != NoRepeat)
+    if (config.readStoresPos && rep != NoRepeat)
       attrDebugStart(id, dataType, Some(io), rep)
 
     dataType match {
@@ -62,7 +62,7 @@ trait EveryReadIsExpression
         handleAssignment(id, expr, rep, isRaw)
     }
 
-    if (debug && rep != NoRepeat)
+    if (config.readStoresPos && rep != NoRepeat)
       attrDebugEnd(id, dataType, io, rep)
   }
 
@@ -131,13 +131,13 @@ trait EveryReadIsExpression
         io
     }
     val expr = parseExpr(dataType, dataType, newIO, defEndian)
-    if (!debug) {
+    if (config.autoRead) {
       handleAssignment(id, expr, rep, false)
     } else {
-      // Debug mode requires one to actually call "_read" method on constructed user type,
-      // and this must be done as a separate statement - or else exception handler would
-      // blast the whole structure, not only this element. This, in turn, makes us assign
-      // constructed element to a temporary variable in case on repetitions
+      // Disabled autoRead requires one to actually call `_read` method on constructed
+      // user type, and this must be done as a separate statement - or else exception
+      // handler would blast the whole structure, not only this element. This, in turn,
+      // makes us assign constructed element to a temporary variable in case of arrays.
       rep match {
         case NoRepeat =>
           handleAssignmentSimple(id, expr)
@@ -204,7 +204,7 @@ trait EveryReadIsExpression
   def userTypeDebugRead(id: String): Unit = {}
 
   def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = {
-    if (debug)
+    if (config.readStoresPos)
       attrDebugStart(instName, dataType, None, NoRepeat)
     handleAssignmentSimple(instName, expression(value))
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index f4929ff5c..63a80fbd5 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -10,7 +10,7 @@ import scala.collection.mutable.ListBuffer
 
 abstract class LanguageCompiler(
   typeProvider: ClassTypeProvider,
-  config: RuntimeConfig
+  val config: RuntimeConfig
 ) extends SwitchOps {
 
   val translator: AbstractTranslator
@@ -51,7 +51,7 @@ abstract class LanguageCompiler(
     */
   def innerDocstrings: Boolean = false
 
-  def debug = config.debug
+  def debug: Boolean = !config.autoRead && config.readStoresPos
 
   def indent: String
   def outFileName(topClassName: String): String

From 7c9e6b7de33d61bb70cc86c7a3cd55c2bbb8076c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 2 Sep 2018 03:41:23 +0100
Subject: [PATCH 165/230] CppCompiler: implemented --no-auto-read support

---
 .../main/scala/io/kaitai/struct/languages/CppCompiler.scala | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index c35ec6dca..4006beaa3 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -578,6 +578,9 @@ class CppCompiler(
     outSrc.puts(s"${privateMemberName(id)} = $expr;")
   }
 
+  override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit =
+    outSrc.puts(s"${kaitaiType2NativeType(dataType)} $id = $expr;")
+
   override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = {
     dataType match {
       case t: ReadableType =>
@@ -624,6 +627,9 @@ class CppCompiler(
     expr2
   }
 
+  override def userTypeDebugRead(id: String): Unit =
+    outSrc.puts(s"$id->_read();")
+
   /**
     * Designates switch mode. If false, we're doing real switch-case for this
     * attribute. If true, we're doing if-based emulation.

From 619037b4ecea7560bd208d9cf8056892ec65d04e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 2 Sep 2018 03:44:16 +0100
Subject: [PATCH 166/230] Generating no userTypeDebugRead at all is always a
 source of run-time data corruption -> let's at least make it compile-time
 error

---
 .../struct/languages/components/EveryReadIsExpression.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
index 41ae56128..6814469b6 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
@@ -201,7 +201,7 @@ trait EveryReadIsExpression
 
   def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String
   def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean): String
-  def userTypeDebugRead(id: String): Unit = {}
+  def userTypeDebugRead(id: String): Unit = ???
 
   def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr): Unit = {
     if (config.readStoresPos)

From f9d1b678b5d32555999b87d5b8e2a8061ef4b32d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 2 Sep 2018 03:48:05 +0100
Subject: [PATCH 167/230] PythonCompiler: fixed debug_user_array

---
 .../main/scala/io/kaitai/struct/languages/PythonCompiler.scala | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index 3333f5e08..93f351b87 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -393,6 +393,9 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     expr2
   }
 
+  override def userTypeDebugRead(id: String): Unit =
+    out.puts(s"$id._read()")
+
   override def switchStart(id: Identifier, on: Ast.expr): Unit = {
     out.puts(s"_on = ${expression(on)}")
   }

From 99318c024ed6242fb56a64ccb6d4df4311f4810c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 2 Sep 2018 19:48:11 +0100
Subject: [PATCH 168/230] PHPCompiler: fixed to work in --no-auto-read mode

---
 .../scala/io/kaitai/struct/languages/PHPCompiler.scala | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
index 2bef260ad..9b4847b90 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
@@ -133,8 +133,10 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case Some(e) => s"${e.toSuffix.toUpperCase}"
       case None => ""
     }
+    val access = if (config.autoRead) "private" else "public"
+
     out.puts
-    out.puts(s"private function _read$suffix() {")
+    out.puts(s"$access function _read$suffix() {")
     out.inc
   }
 
@@ -304,6 +306,9 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"${privateMemberName(id)} = $expr;")
   }
 
+  override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit =
+    out.puts(s"$id = $expr;")
+
   override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = {
     dataType match {
       case t: ReadableType =>
@@ -350,6 +355,9 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     expr2
   }
 
+  override def userTypeDebugRead(id: String): Unit =
+    out.puts(s"$id->_read();")
+
   override def switchStart(id: Identifier, on: Ast.expr): Unit = {
     val onType = translator.detectType(on)
 

From c5564ac7547fe67626811e4b51824b786cdf409d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 2 Sep 2018 20:25:23 +0100
Subject: [PATCH 169/230] CSharpCompiler: implemented support for
 --no-auto-read

---
 .../io/kaitai/struct/languages/CSharpCompiler.scala | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
index db80b797f..73b7e9e2f 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
@@ -2,10 +2,10 @@ package io.kaitai.struct.languages
 
 import io.kaitai.struct._
 import io.kaitai.struct.datatype.DataType._
-import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian}
+import io.kaitai.struct.datatype._
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.exprlang.Ast.expr
-import io.kaitai.struct.format.{RepeatUntil, _}
+import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
 import io.kaitai.struct.translators.{CSharpTranslator, TypeDetector}
 
@@ -341,9 +341,11 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("}")
   }
 
-  override def handleAssignmentSimple(id: Identifier, expr: String): Unit = {
+  override def handleAssignmentSimple(id: Identifier, expr: String): Unit =
     out.puts(s"${privateMemberName(id)} = $expr;")
-  }
+
+  override def handleAssignmentTempVar(dataType: DataType, id: String, expr: String): Unit =
+    out.puts(s"${kaitaiType2NativeType(dataType)} $id = $expr;")
 
   override def parseExpr(dataType: DataType, assignType: DataType, io: String, defEndian: Option[FixedEndian]): String = {
     dataType match {
@@ -391,6 +393,9 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     expr2
   }
 
+  override def userTypeDebugRead(id: String): Unit =
+    out.puts(s"$id._read();")
+
   /**
     * Designates switch mode. If false, we're doing real switch-case for this
     * attribute. If true, we're doing if-based emulation.

From ea959dba657c3e9a1d04ef755d4fbbe3efc9e125 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 8 Oct 2018 12:50:31 +0100
Subject: [PATCH 170/230] Fix GraphvizClassCompiler to handle all special names
 (like _io, _index, etc) properly

---
 .../io/kaitai/struct/GraphvizClassCompiler.scala      | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
index 83d7b725f..dd2218900 100644
--- a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
@@ -333,11 +333,14 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
       case SwitchType.ELSE_CONST =>
         // "_" is a special const for
         List()
-      case expr.Name(Ast.identifier("_io")) =>
-        // "_io" is a special const too
-        List()
       case expr.Name(id) =>
-        List(resolveLocalNode(id.name))
+        if (id.name.charAt(0) == '_') {
+          // other special consts like "_io", "_index", etc
+          List()
+        } else {
+          // this must be local name, resolve it
+          List(resolveLocalNode(id.name))
+        }
       case expr.List(elts) =>
         elts.flatMap(affectedVars).toList
     }

From 8bd15dfd9a04d2b520aa2a8884010f9e7527696c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 8 Oct 2018 20:30:03 +0100
Subject: [PATCH 171/230] Added C++ compiler namespaces support

---
 jvm/src/main/scala/io/kaitai/struct/JavaMain.scala |  4 ++++
 .../scala/io/kaitai/struct/RuntimeConfig.scala     |  2 ++
 .../io/kaitai/struct/languages/CppCompiler.scala   | 14 ++++++++++++++
 3 files changed, 20 insertions(+)

diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
index 97bc2fbe7..7dea6a958 100644
--- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
+++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
@@ -64,6 +64,10 @@ object JavaMain {
         c.copy(importPaths = c.importPaths ++ x.split(File.pathSeparatorChar))
       } text(".ksy library search path(s) for imports (see also KSPATH env variable)")
 
+      opt[String]("cpp-namespace") valueName("<namespace>") action { (x, c) =>
+        c.copy(runtime = c.runtime.copy(cppNamespace = x.split("::").toList))
+      } text("C++ namespace (C++ only, default: none)")
+
       opt[String]("go-package") valueName("<package>") action { (x, c) =>
         c.copy(runtime = c.runtime.copy(goPackage = x))
       } text("Go package (Go only, default: none)")
diff --git a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
index f31147725..d16f00840 100644
--- a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
@@ -17,6 +17,7 @@ package io.kaitai.struct
   *                    "opaque" type, i.e. an external KaitaiStruct-compatible type
   *                    defined somewhere else. If false, it will be reported as
   *                    precompile error.
+  * @param cppNamespace C++ namespace
   * @param goPackage Go package name
   * @param javaPackage Java package name
   * @param javaFromFileClass Java class to be invoked in `fromFile` helper methods
@@ -28,6 +29,7 @@ case class RuntimeConfig(
   autoRead: Boolean = true,
   readStoresPos: Boolean = false,
   opaqueTypes: Boolean = false,
+  cppNamespace: List[String] = List(),
   goPackage: String = "",
   javaPackage: String = "",
   javaFromFileClass: String = "io.kaitai.struct.ByteBufferKaitaiStream",
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 4006beaa3..cc7da984c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -76,11 +76,25 @@ class CppCompiler(
         KSVersion.minimalRuntime + " or later is required\""
     )
     outHdr.puts("#endif")
+
+    config.cppNamespace.foreach { (namespace) =>
+      outSrc.puts(s"namespace $namespace {")
+      outSrc.inc
+      outHdr.puts(s"namespace $namespace {")
+      outHdr.inc
+    }
   }
 
   override def fileFooter(topClassName: String): Unit = {
     outHdr.puts
     outHdr.puts(s"#endif  // ${defineName(topClassName)}")
+
+    config.cppNamespace.foreach { (_) =>
+      outSrc.dec
+      outSrc.puts("}")
+      outHdr.dec
+      outHdr.puts("}")
+    }
   }
 
   override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = {

From b8b2da61bfbf6f79b6756f3152ccf30727cc9f3b Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 23 Oct 2018 08:22:58 +0100
Subject: [PATCH 172/230] More docs in code

---
 .../io/kaitai/struct/ClassCompiler.scala      | 37 +++++++++++++++++--
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index f30cd317a..aa3f7bfc7 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -120,10 +120,10 @@ class ClassCompiler(
   }
 
   /**
-    * Compiles destructor for a given class. It should clean up everything
+    * Compiles destructor for a given type. It should clean up everything
     * (i.e. every applicable allocated seq / instance attribute variables, and
     * any extra attribute variables, if they were used).
-    * @param curClass current class to generate code for
+    * @param curClass current type to generate code for
     */
   def compileDestructor(curClass: ClassSpec) = {
     lang.classDestructorHeader(curClass.name, curClass.parentType, topClassName)
@@ -137,6 +137,11 @@ class ClassCompiler(
     lang.classDestructorFooter
   }
 
+  /**
+    * Iterates over a given list of attributes and generates attribute
+    * declarations for each of them.
+    * @param attrs attribute list to traverse
+    */
   def compileAttrDeclarations(attrs: List[MemberSpec]): Unit = {
     attrs.foreach { (attr) =>
       val isNullable = if (lang.switchBytesOnlyAsRaw) {
@@ -153,7 +158,7 @@ class ClassCompiler(
     * readers (AKA getters) for each of them.
     * @param attrs attribute list to traverse
     */
-  def compileAttrReaders(attrs: List[MemberSpec]): Unit =
+  def compileAttrReaders(attrs: List[MemberSpec]): Unit = {
     attrs.foreach { (attr) =>
       // FIXME: Python should have some form of attribute docs too
       if (!attr.doc.isEmpty && !lang.innerDocstrings)
@@ -165,7 +170,22 @@ class ClassCompiler(
       }
       lang.attributeReader(attr.id, attr.dataTypeComposite, isNullable)
     }
+  }
 
+  /**
+    * Compiles everything related to "eager reading" for a given list of
+    * sequence attributes and endianness. Depending on endianness:
+    *
+    * * For types known to have fixed endianness, we do just "_read" method.
+    * * For types with ambiguous endianness, we'll do `_read` + "_read_le" +
+    *   "_read_be" methods. If endianness needs to be calculated, we'll perform
+    *   that calculation in "_read". If it's inherited, then we'll just make
+    *   decision based on that inherited setting.
+    *
+    * @param seq list of sequence attributes
+    * @param extraAttrs
+    * @param endian endianness setting
+    */
   def compileEagerRead(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec], endian: Option[Endianness]): Unit = {
     endian match {
       case None | Some(_: FixedEndian) =>
@@ -190,6 +210,11 @@ class ClassCompiler(
 
   val IS_LE_ID = SpecialIdentifier("_is_le")
 
+  /**
+    * Compiles endianness calculation procedure and stores result in a special
+    * attribute [[IS_LE_ID]]. Typically occurs as part of "_read" method.
+    * @param ce calculated endianness specification
+    */
   def compileCalcEndian(ce: CalcEndian): Unit = {
     def renderProc(result: FixedEndian): Unit = {
       val v = Ast.expr.Bool(result == LittleEndian)
@@ -228,12 +253,16 @@ class ClassCompiler(
     }
   }
 
+  /**
+    * Compiles all enums specifications for a given type.
+    * @param curClass current type to generate code for
+    */
   def compileEnums(curClass: ClassSpec): Unit =
     curClass.enums.foreach { case(_, enumColl) => compileEnum(curClass, enumColl) }
 
   /**
     * Compile subclasses for a given class.
-    * @param curClass current class to generate code for
+    * @param curClass current type to generate code for
     */
   def compileSubclasses(curClass: ClassSpec): Unit =
     curClass.types.foreach { case (_, intClass) => compileClass(intClass) }

From a3e5b18ead7c2adf63dd25237214099b3f83e05f Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 24 Oct 2018 03:13:27 +0100
Subject: [PATCH 173/230] Fix TypeDetector to make enums properly assignable

---
 .../main/scala/io/kaitai/struct/translators/TypeDetector.scala | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index d304601b9..e1acb3fc0 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -412,6 +412,9 @@ object TypeDetector {
             case (_, _) =>
               false
           }
+        case (t1: EnumType, t2: EnumType) =>
+          // enums are assignable if their enumSpecs match
+          t1.enumSpec.get == t2.enumSpec.get
         case (_, _) => false
       }
     }

From ca415b29b830ab34de283efdf09d796bd6648f05 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 24 Oct 2018 10:06:21 +0100
Subject: [PATCH 174/230] Moved ExtraAttrs-related collector from
 GoClassCompiler to ExtraAttrs

---
 .../scala/io/kaitai/struct/GoClassCompiler.scala  | 15 +--------------
 .../struct/languages/components/ExtraAttrs.scala  | 13 +++++++++++++
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
index bb3e9a0db..99602928e 100644
--- a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
@@ -22,7 +22,7 @@ class GoClassCompiler(
     extraAttrs += AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None))
     extraAttrs += AttrSpec(List(), ParentIdentifier, curClass.parentType)
 
-    extraAttrs ++= getExtraAttrs(curClass)
+    extraAttrs ++= ExtraAttrs.forClassSpec(curClass)
 
     if (!curClass.doc.isEmpty)
       lang.classDoc(curClass.name, curClass.doc)
@@ -90,17 +90,4 @@ class GoClassCompiler(
     lang.instanceReturn(instName)
     lang.instanceFooter
   }
-
-  def getExtraAttrs(curClass: ClassSpec): List[AttrSpec] = {
-    // We want only values of ParseInstances, which are AttrSpecLike.
-    // ValueInstances are ignored, as they can't currently generate
-    // any extra attributes (i.e. no `size`, no `process`, etc)
-    val parseInstances = curClass.instances.values.collect {
-      case inst: AttrLikeSpec => inst
-    }
-
-    (curClass.seq ++ parseInstances).foldLeft(List[AttrSpec]())(
-      (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr)
-    )
-  }
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
index 75f2ba6bd..ed5796707 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
@@ -13,6 +13,19 @@ import io.kaitai.struct.format._
   * * unprocessed / postprocessed byte arrays
   */
 object ExtraAttrs {
+  def forClassSpec(curClass: ClassSpec): List[AttrSpec] = {
+    // We want only values of ParseInstances, which are AttrSpecLike.
+    // ValueInstances are ignored, as they can't currently generate
+    // any extra attributes (i.e. no `size`, no `process`, etc)
+    val parseInstances = curClass.instances.values.collect {
+      case inst: AttrLikeSpec => inst
+    }
+
+    (curClass.seq ++ parseInstances).foldLeft(List[AttrSpec]())(
+      (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr)
+    )
+  }
+
   def forAttr(attr: AttrLikeSpec): List[AttrSpec] =
     forAttr(attr.id, attr.dataType, attr.cond)
 

From 3bc83807c133f14867421b768d8dec4c436e9b2e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 24 Oct 2018 15:34:48 +0100
Subject: [PATCH 175/230] Made all languages work through ExtraAttrs mechanism

---
 .../io/kaitai/struct/ClassCompiler.scala      | 13 +++++++--
 .../io/kaitai/struct/GoClassCompiler.scala    |  2 +-
 .../io/kaitai/struct/RustClassCompiler.scala  |  8 +-----
 .../components/AllocateAndStoreIO.scala       | 14 ++++++++--
 .../components/AllocateIOLocalVar.scala       |  6 ++--
 .../languages/components/ExtraAttrs.scala     | 28 ++++++++++++++-----
 .../components/LanguageCompiler.scala         |  3 +-
 7 files changed, 51 insertions(+), 23 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index aa3f7bfc7..97a9f7272 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -4,8 +4,8 @@ import io.kaitai.struct.CompileLog.FileSuccess
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.datatype._
 import io.kaitai.struct.exprlang.Ast
-import io.kaitai.struct.format._
-import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic}
+import io.kaitai.struct.format.{AttrSpec, _}
+import io.kaitai.struct.languages.components.{ExtraAttrs, LanguageCompiler, LanguageCompilerStatic}
 
 import scala.collection.mutable.ListBuffer
 
@@ -83,7 +83,14 @@ class ClassCompiler(
     compileInstances(curClass, extraAttrs)
 
     // Attributes declarations and readers
-    val allAttrs: List[MemberSpec] = curClass.seq ++ curClass.params ++ extraAttrs
+    val allAttrs: List[MemberSpec] =
+      curClass.seq ++
+      curClass.params ++
+      List(
+        AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None)),
+        AttrSpec(List(), ParentIdentifier, curClass.parentType)
+      ) ++
+      ExtraAttrs.forClassSpec(curClass, lang)
     compileAttrDeclarations(allAttrs)
     compileAttrReaders(allAttrs)
 
diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
index 99602928e..80414a1a8 100644
--- a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
@@ -22,7 +22,7 @@ class GoClassCompiler(
     extraAttrs += AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None))
     extraAttrs += AttrSpec(List(), ParentIdentifier, curClass.parentType)
 
-    extraAttrs ++= ExtraAttrs.forClassSpec(curClass)
+    extraAttrs ++= ExtraAttrs.forClassSpec(curClass, lang)
 
     if (!curClass.doc.isEmpty)
       lang.classDoc(curClass.name, curClass.doc)
diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
index 7de42a0c7..697fdb415 100644
--- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
@@ -22,7 +22,7 @@ class RustClassCompiler(
     extraAttrs += AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None))
     extraAttrs += AttrSpec(List(), ParentIdentifier, curClass.parentType)
 
-    extraAttrs ++= getExtraAttrs(curClass)
+    extraAttrs ++= ExtraAttrs.forClassSpec(curClass, lang)
 
     if (!curClass.doc.isEmpty)
       lang.classDoc(curClass.name, curClass.doc)
@@ -101,10 +101,4 @@ class RustClassCompiler(
     lang.instanceReturn(instName)
     lang.instanceFooter
   }
-
-  def getExtraAttrs(curClass: ClassSpec): List[AttrSpec] = {
-    curClass.seq.foldLeft(List[AttrSpec]())(
-      (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr)
-    )
-  }
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
index cac6a7fdc..2aedca65d 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
@@ -1,6 +1,7 @@
 package io.kaitai.struct.languages.components
 
-import io.kaitai.struct.format.{AttrSpec, Identifier, RepeatSpec}
+import io.kaitai.struct.datatype.DataType.{ArrayType, KaitaiStreamType}
+import io.kaitai.struct.format._
 
 import scala.collection.mutable.ListBuffer
 
@@ -9,6 +10,15 @@ import scala.collection.mutable.ListBuffer
   * at. This is used for languages without garbage collection that need to
   * keep track of allocated IOs.
   */
-trait AllocateAndStoreIO {
+trait AllocateAndStoreIO extends ExtraAttrs {
   def allocateIO(id: Identifier, rep: RepeatSpec, extraAttrs: ListBuffer[AttrSpec]): String
+
+  override def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec] = {
+    val ioId = IoStorageIdentifier(id)
+    val ioType = rep match {
+      case NoRepeat => KaitaiStreamType
+      case _ => ArrayType(KaitaiStreamType)
+    }
+    List(AttrSpec(List(), ioId, ioType))
+  }
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala
index 6334d759a..fd84dce12 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateIOLocalVar.scala
@@ -1,11 +1,13 @@
 package io.kaitai.struct.languages.components
 
-import io.kaitai.struct.format.{Identifier, RepeatSpec}
+import io.kaitai.struct.format.{AttrSpec, Identifier, RepeatSpec}
 
 /**
   * Allocates new auxiliary IOs as local vars - no references saved and thus
   * probably garbage collector will deal with them.
   */
-trait AllocateIOLocalVar {
+trait AllocateIOLocalVar extends ExtraAttrs {
   def allocateIO(varName: Identifier, rep: RepeatSpec): String
+
+  override def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec] = List()
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
index ed5796707..563fa3f32 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
@@ -4,6 +4,14 @@ import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.format._
 
+/**
+  * Trait to be implemented by all [[LanguageCompiler]] compilers: supplies extra attributes
+  * when we'll be allocating new IOs.
+  */
+trait ExtraAttrs {
+  def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec]
+}
+
 /**
   * Generates list of extra attributes required to store intermediate /
   * virtual stuff for every attribute like:
@@ -13,7 +21,7 @@ import io.kaitai.struct.format._
   * * unprocessed / postprocessed byte arrays
   */
 object ExtraAttrs {
-  def forClassSpec(curClass: ClassSpec): List[AttrSpec] = {
+  def forClassSpec(curClass: ClassSpec, compiler: ExtraAttrs): List[AttrSpec] = {
     // We want only values of ParseInstances, which are AttrSpecLike.
     // ValueInstances are ignored, as they can't currently generate
     // any extra attributes (i.e. no `size`, no `process`, etc)
@@ -22,26 +30,32 @@ object ExtraAttrs {
     }
 
     (curClass.seq ++ parseInstances).foldLeft(List[AttrSpec]())(
-      (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr)
+      (attrs, attr) => attrs ++ ExtraAttrs.forAttr(attr, compiler)
     )
   }
 
-  def forAttr(attr: AttrLikeSpec): List[AttrSpec] =
-    forAttr(attr.id, attr.dataType, attr.cond)
+  def forAttr(attr: AttrLikeSpec, compiler: ExtraAttrs): Iterable[AttrSpec] =
+    forAttr(attr.id, attr.dataType, attr.cond, compiler)
 
   private
-  def forAttr(id: Identifier, dataType: DataType, condSpec: ConditionalSpec): List[AttrSpec] = {
+  def forAttr(id: Identifier, dataType: DataType, condSpec: ConditionalSpec, compiler: ExtraAttrs): Iterable[AttrSpec] = {
     dataType match {
       case bt: BytesType =>
-        bt.process match {
+        val rawIdAttrs = bt.process match {
           case None => List()
           case Some(_) =>
             val rawId = RawIdentifier(id)
             List(AttrSpec(List(), rawId, bt, condSpec))
         }
+        val ioIdAttrs = compiler.extraAttrForIO(id, condSpec.repeat)
+        rawIdAttrs ++ ioIdAttrs
       case utb: UserTypeFromBytes =>
         val rawId = RawIdentifier(id)
-        List(AttrSpec(List(), rawId, utb.bytes, condSpec)) ++ forAttr(rawId, utb.bytes, condSpec)
+        List(AttrSpec(List(), rawId, utb.bytes, condSpec)) ++ forAttr(rawId, utb.bytes, condSpec, compiler)
+      case st: SwitchType =>
+        st.cases.flatMap { case (_, caseType) =>
+          forAttr(id, caseType, condSpec, compiler)
+        }.toList.distinct
       case _ =>
         List()
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index 63a80fbd5..dcbe0618c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -11,7 +11,8 @@ import scala.collection.mutable.ListBuffer
 abstract class LanguageCompiler(
   typeProvider: ClassTypeProvider,
   val config: RuntimeConfig
-) extends SwitchOps {
+) extends SwitchOps
+  with ExtraAttrs {
 
   val translator: AbstractTranslator
 

From 27ef4bf1b2b346c761f41c773f4083eef1057b97 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 24 Oct 2018 16:08:50 +0100
Subject: [PATCH 176/230] Remove old mutable extraAttrs solution everywhere -
 it is replaced with new immutable one (ExtraAttrs)

---
 .../io/kaitai/struct/ClassCompiler.scala      | 41 ++++++++-----------
 .../io/kaitai/struct/GoClassCompiler.scala    | 19 ++++-----
 .../io/kaitai/struct/RustClassCompiler.scala  | 16 ++++----
 .../kaitai/struct/languages/CppCompiler.scala |  9 ++--
 .../components/AllocateAndStoreIO.scala       |  2 +-
 .../languages/components/CommonReads.scala    | 24 +++++------
 .../components/EveryReadIsExpression.scala    | 30 +++++---------
 .../struct/languages/components/GoReads.scala | 11 ++---
 .../components/LanguageCompiler.scala         |  2 +-
 9 files changed, 65 insertions(+), 89 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index 97a9f7272..b52cbb158 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -7,8 +7,6 @@ import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.{AttrSpec, _}
 import io.kaitai.struct.languages.components.{ExtraAttrs, LanguageCompiler, LanguageCompilerStatic}
 
-import scala.collection.mutable.ListBuffer
-
 class ClassCompiler(
   classSpecs: ClassSpecs,
   val topClass: ClassSpec,
@@ -51,10 +49,6 @@ class ClassCompiler(
     if (lang.innerDocstrings)
       compileClassDoc(curClass)
 
-    val extraAttrs = ListBuffer[AttrSpec]()
-    extraAttrs += AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None))
-    extraAttrs += AttrSpec(List(), ParentIdentifier, curClass.parentType)
-
     // Forward declarations for recursive types
     curClass.types.foreach { case (typeName, _) => lang.classForwardDeclaration(List(typeName)) }
 
@@ -68,7 +62,7 @@ class ClassCompiler(
     compileConstructor(curClass)
 
     // Read method(s)
-    compileEagerRead(curClass.seq, extraAttrs, curClass.meta.endian)
+    compileEagerRead(curClass.seq, curClass.meta.endian)
 
     // Destructor
     compileDestructor(curClass)
@@ -80,7 +74,7 @@ class ClassCompiler(
       provider.nowClass = curClass
     }
 
-    compileInstances(curClass, extraAttrs)
+    compileInstances(curClass)
 
     // Attributes declarations and readers
     val allAttrs: List[MemberSpec] =
@@ -190,28 +184,27 @@ class ClassCompiler(
     *   decision based on that inherited setting.
     *
     * @param seq list of sequence attributes
-    * @param extraAttrs
     * @param endian endianness setting
     */
-  def compileEagerRead(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec], endian: Option[Endianness]): Unit = {
+  def compileEagerRead(seq: List[AttrSpec], endian: Option[Endianness]): Unit = {
     endian match {
       case None | Some(_: FixedEndian) =>
-        compileSeqProc(seq, extraAttrs, None)
+        compileSeqProc(seq, None)
       case Some(ce: CalcEndian) =>
         lang.readHeader(None, false)
         compileCalcEndian(ce)
         lang.runReadCalc()
         lang.readFooter()
 
-        compileSeqProc(seq, extraAttrs, Some(LittleEndian))
-        compileSeqProc(seq, extraAttrs, Some(BigEndian))
+        compileSeqProc(seq, Some(LittleEndian))
+        compileSeqProc(seq, Some(BigEndian))
       case Some(InheritedEndian) =>
         lang.readHeader(None, false)
         lang.runReadCalc()
         lang.readFooter()
 
-        compileSeqProc(seq, extraAttrs, Some(LittleEndian))
-        compileSeqProc(seq, extraAttrs, Some(BigEndian))
+        compileSeqProc(seq, Some(LittleEndian))
+        compileSeqProc(seq, Some(BigEndian))
     }
   }
 
@@ -234,28 +227,26 @@ class ClassCompiler(
   /**
     * Compiles seq reading method (complete with header and footer).
     * @param seq sequence of attributes
-    * @param extraAttrs extra attributes to be allocated
     * @param defEndian default endianness
     */
-  def compileSeqProc(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec], defEndian: Option[FixedEndian]) = {
+  def compileSeqProc(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
     lang.readHeader(defEndian, seq.isEmpty)
-    compileSeq(seq, extraAttrs, defEndian)
+    compileSeq(seq, defEndian)
     lang.readFooter()
   }
 
   /**
     * Compiles seq reading method body (only reading statements).
     * @param seq sequence of attributes
-    * @param extraAttrs extra attributes to be allocated
     * @param defEndian default endianness
     */
-  def compileSeq(seq: List[AttrSpec], extraAttrs: ListBuffer[AttrSpec], defEndian: Option[FixedEndian]) = {
+  def compileSeq(seq: List[AttrSpec], defEndian: Option[FixedEndian]) = {
     var wasUnaligned = false
     seq.foreach { (attr) =>
       val nowUnaligned = isUnalignedBits(attr.dataType)
       if (wasUnaligned && !nowUnaligned)
         lang.alignToByte(lang.normalIO)
-      lang.attrParse(attr, attr.id, extraAttrs, defEndian)
+      lang.attrParse(attr, attr.id, defEndian)
       wasUnaligned = nowUnaligned
     }
   }
@@ -274,13 +265,13 @@ class ClassCompiler(
   def compileSubclasses(curClass: ClassSpec): Unit =
     curClass.types.foreach { case (_, intClass) => compileClass(intClass) }
 
-  def compileInstances(curClass: ClassSpec, extraAttrs: ListBuffer[AttrSpec]) = {
+  def compileInstances(curClass: ClassSpec) = {
     curClass.instances.foreach { case (instName, instSpec) =>
-      compileInstance(curClass.name, instName, instSpec, extraAttrs, curClass.meta.endian)
+      compileInstance(curClass.name, instName, instSpec, curClass.meta.endian)
     }
   }
 
-  def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, extraAttrs: ListBuffer[AttrSpec], endian: Option[Endianness]): Unit = {
+  def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = {
     // Determine datatype
     val dataType = instSpec.dataTypeComposite
 
@@ -299,7 +290,7 @@ class ClassCompiler(
         lang.instanceCalculate(instName, dataType, vi.value)
         lang.attrParseIfFooter(vi.ifExpr)
       case i: ParseInstanceSpec =>
-        lang.attrParse(i, instName, extraAttrs, endian)
+        lang.attrParse(i, instName, endian)
     }
 
     lang.instanceSetCalculated(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
index 80414a1a8..db5a750a1 100644
--- a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
@@ -17,12 +17,11 @@ class GoClassCompiler(
   override def compileClass(curClass: ClassSpec): Unit = {
     provider.nowClass = curClass
 
-    val extraAttrs = ListBuffer[AttrSpec]()
-    extraAttrs += AttrSpec(List(), IoIdentifier, KaitaiStreamType)
-    extraAttrs += AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None))
-    extraAttrs += AttrSpec(List(), ParentIdentifier, curClass.parentType)
-
-    extraAttrs ++= ExtraAttrs.forClassSpec(curClass, lang)
+    val extraAttrs = List(
+      AttrSpec(List(), IoIdentifier, KaitaiStreamType),
+      AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None)),
+      AttrSpec(List(), ParentIdentifier, curClass.parentType)
+    ) ++ ExtraAttrs.forClassSpec(curClass, lang)
 
     if (!curClass.doc.isEmpty)
       lang.classDoc(curClass.name, curClass.doc)
@@ -41,7 +40,7 @@ class GoClassCompiler(
     // Constructor = Read() function
     compileReadFunction(curClass)
 
-    compileInstances(curClass, extraAttrs)
+    compileInstances(curClass)
 
     compileAttrReaders(curClass.seq ++ extraAttrs)
 
@@ -62,11 +61,11 @@ class GoClassCompiler(
       case Some(fe: FixedEndian) => Some(fe)
       case _ => None
     }
-    compileSeq(curClass.seq, new ListBuffer[AttrSpec], defEndian)
+    compileSeq(curClass.seq, defEndian)
     lang.classConstructorFooter
   }
 
-  override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, extraAttrs: ListBuffer[AttrSpec], endian: Option[Endianness]): Unit = {
+  override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = {
     // FIXME: support calculated endianness
 
     // Determine datatype
@@ -83,7 +82,7 @@ class GoClassCompiler(
         lang.instanceCalculate(instName, dataType, vi.value)
         lang.attrParseIfFooter(vi.ifExpr)
       case i: ParseInstanceSpec =>
-        lang.attrParse(i, instName, extraAttrs, None) // FIXME
+        lang.attrParse(i, instName, None) // FIXME
     }
 
     lang.instanceSetCalculated(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
index 697fdb415..f28f31c2b 100644
--- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
@@ -36,9 +36,9 @@ class RustClassCompiler(
     }
     
     // Constructor = Read() function
-    compileReadFunction(curClass, extraAttrs)
+    compileReadFunction(curClass)
     
-    compileInstances(curClass, extraAttrs)
+    compileInstances(curClass)
 
     compileAttrReaders(curClass.seq ++ extraAttrs)
     lang.classFooter(curClass.name)
@@ -49,7 +49,7 @@ class RustClassCompiler(
     compileSubclasses(curClass)
   }
 
-  def compileReadFunction(curClass: ClassSpec, extraAttrs: ListBuffer[AttrSpec]) = {
+  def compileReadFunction(curClass: ClassSpec) = {
     lang.classConstructorHeader(
       curClass.name,
       curClass.parentType,
@@ -66,18 +66,18 @@ class RustClassCompiler(
     
     lang.readHeader(defEndian, false)
     
-    compileSeq(curClass.seq, extraAttrs, defEndian)
+    compileSeq(curClass.seq, defEndian)
     lang.classConstructorFooter
   }
 
-  override def compileInstances(curClass: ClassSpec, extraAttrs: ListBuffer[AttrSpec]) = {
+  override def compileInstances(curClass: ClassSpec) = {
     lang.instanceDeclHeader(curClass.name)
     curClass.instances.foreach { case (instName, instSpec) =>
-      compileInstance(curClass.name, instName, instSpec, extraAttrs, curClass.meta.endian)
+      compileInstance(curClass.name, instName, instSpec, curClass.meta.endian)
     }
   }
 
-  override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, extraAttrs: ListBuffer[AttrSpec], endian: Option[Endianness]): Unit = {
+  override def compileInstance(className: List[String], instName: InstanceIdentifier, instSpec: InstanceSpec, endian: Option[Endianness]): Unit = {
     // FIXME: support calculated endianness
 
     // Determine datatype
@@ -94,7 +94,7 @@ class RustClassCompiler(
         lang.instanceCalculate(instName, dataType, vi.value)
         lang.attrParseIfFooter(vi.ifExpr)
       case i: ParseInstanceSpec =>
-        lang.attrParse(i, instName, extraAttrs, None) // FIXME
+        lang.attrParse(i, instName, None) // FIXME
     }
 
     lang.instanceSetCalculated(instName)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index cc7da984c..911c11246 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -432,7 +432,7 @@ class CppCompiler(
     }
   }
 
-  override def allocateIO(id: Identifier, rep: RepeatSpec, extraAttrs: ListBuffer[AttrSpec]): String = {
+  override def allocateIO(id: Identifier, rep: RepeatSpec): String = {
     val memberName = privateMemberName(id)
     val ioId = IoStorageIdentifier(id)
 
@@ -444,18 +444,17 @@ class CppCompiler(
 
     val newStream = s"new $kstreamName($args)"
 
-    val (ioType, ioName) = rep match {
+    val ioName = rep match {
       case NoRepeat =>
         outSrc.puts(s"${privateMemberName(ioId)} = $newStream;")
-        (KaitaiStreamType, privateMemberName(ioId))
+        privateMemberName(ioId)
       case _ =>
         val localIO = s"io_${idToStr(id)}"
         outSrc.puts(s"$kstreamName* $localIO = $newStream;")
         outSrc.puts(s"${privateMemberName(ioId)}->push_back($localIO);")
-        (ArrayType(KaitaiStreamType), localIO)
+        localIO
     }
 
-    Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), ioId, ioType))
     ioName
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
index 2aedca65d..acd56ebe2 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
@@ -11,7 +11,7 @@ import scala.collection.mutable.ListBuffer
   * keep track of allocated IOs.
   */
 trait AllocateAndStoreIO extends ExtraAttrs {
-  def allocateIO(id: Identifier, rep: RepeatSpec, extraAttrs: ListBuffer[AttrSpec]): String
+  def allocateIO(id: Identifier, rep: RepeatSpec): String
 
   override def extraAttrForIO(id: Identifier, rep: RepeatSpec): List[AttrSpec] = {
     val ioId = IoStorageIdentifier(id)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala
index 8bf4b7a5c..d938b7b98 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/CommonReads.scala
@@ -5,10 +5,8 @@ import io.kaitai.struct.datatype.DataType.{SwitchType, UserTypeFromBytes}
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format._
 
-import scala.collection.mutable.ListBuffer
-
 trait CommonReads extends LanguageCompiler {
-  override def attrParse(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec], defEndian: Option[Endianness]): Unit = {
+  override def attrParse(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit = {
     attrParseIfHeader(id, attr.cond.ifExpr)
 
     // Manage IO & seeking for ParseInstances
@@ -34,13 +32,13 @@ trait CommonReads extends LanguageCompiler {
     defEndian match {
       case Some(_: CalcEndian) | Some(InheritedEndian) =>
         attrParseHybrid(
-          () => attrParse0(id, attr, io, extraAttrs, Some(LittleEndian)),
-          () => attrParse0(id, attr, io, extraAttrs, Some(BigEndian))
+          () => attrParse0(id, attr, io, Some(LittleEndian)),
+          () => attrParse0(id, attr, io, Some(BigEndian))
         )
       case None =>
-        attrParse0(id, attr, io, extraAttrs, None)
+        attrParse0(id, attr, io, None)
       case Some(fe: FixedEndian) =>
-        attrParse0(id, attr, io, extraAttrs, Some(fe))
+        attrParse0(id, attr, io, Some(fe))
     }
 
     if (config.readStoresPos)
@@ -57,26 +55,26 @@ trait CommonReads extends LanguageCompiler {
     attrParseIfFooter(attr.cond.ifExpr)
   }
 
-  def attrParse0(id: Identifier, attr: AttrLikeSpec, io: String, extraAttrs: ListBuffer[AttrSpec], defEndian: Option[FixedEndian]): Unit = {
+  def attrParse0(id: Identifier, attr: AttrLikeSpec, io: String, defEndian: Option[FixedEndian]): Unit = {
     attr.cond.repeat match {
       case RepeatEos =>
         condRepeatEosHeader(id, io, attr.dataType, needRaw(attr.dataType))
-        attrParse2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false, defEndian)
+        attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian)
         condRepeatEosFooter
       case RepeatExpr(repeatExpr: Ast.expr) =>
         condRepeatExprHeader(id, io, attr.dataType, needRaw(attr.dataType), repeatExpr)
-        attrParse2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false, defEndian)
+        attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian)
         condRepeatExprFooter
       case RepeatUntil(untilExpr: Ast.expr) =>
         condRepeatUntilHeader(id, io, attr.dataType, needRaw(attr.dataType), untilExpr)
-        attrParse2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false, defEndian)
+        attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian)
         condRepeatUntilFooter(id, io, attr.dataType, needRaw(attr.dataType), untilExpr)
       case NoRepeat =>
-        attrParse2(id, attr.dataType, io, extraAttrs, attr.cond.repeat, false, defEndian)
+        attrParse2(id, attr.dataType, io, attr.cond.repeat, false, defEndian)
     }
   }
 
-  def attrParse2(id: Identifier, dataType: DataType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, isRaw: Boolean, defEndian: Option[FixedEndian], assignType: Option[DataType] = None): Unit
+  def attrParse2(id: Identifier, dataType: DataType, io: String, rep: RepeatSpec, isRaw: Boolean, defEndian: Option[FixedEndian], assignType: Option[DataType] = None): Unit
 
   def needRaw(dataType: DataType): Boolean = {
     dataType match {
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
index 6814469b6..08ee46d99 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/EveryReadIsExpression.scala
@@ -25,7 +25,6 @@ trait EveryReadIsExpression
     id: Identifier,
     dataType: DataType,
     io: String,
-    extraAttrs: ListBuffer[AttrSpec],
     rep: RepeatSpec,
     isRaw: Boolean,
     defEndian: Option[FixedEndian],
@@ -40,9 +39,9 @@ trait EveryReadIsExpression
       case FixedBytesType(c, _) =>
         attrFixedContentsParse(id, c)
       case t: UserType =>
-        attrUserTypeParse(id, t, io, extraAttrs, rep, defEndian)
+        attrUserTypeParse(id, t, io, rep, defEndian)
       case t: BytesType =>
-        attrBytesTypeParse(id, t, io, extraAttrs, rep, isRaw)
+        attrBytesTypeParse(id, t, io, rep, isRaw)
       case st: SwitchType =>
         val isNullable = if (switchBytesOnlyAsRaw) {
           st.isNullableSwitchRaw
@@ -50,7 +49,7 @@ trait EveryReadIsExpression
           st.isNullable
         }
 
-        attrSwitchTypeParse(id, st.on, st.cases, io, extraAttrs, rep, defEndian, isNullable, st.combinedType)
+        attrSwitchTypeParse(id, st.on, st.cases, io, rep, defEndian, isNullable, st.combinedType)
       case t: StrFromBytesType =>
         val expr = translator.bytesToStr(parseExprBytes(t.bytes, io), Ast.expr.Str(t.encoding))
         handleAssignment(id, expr, rep, isRaw)
@@ -70,17 +69,13 @@ trait EveryReadIsExpression
     id: Identifier,
     dataType: BytesType,
     io: String,
-    extraAttrs: ListBuffer[AttrSpec],
     rep: RepeatSpec,
     isRaw: Boolean
   ): Unit = {
     // use intermediate variable name, if we'll be doing post-processing
     val rawId = dataType.process match {
       case None => id
-      case Some(_) =>
-        val rawId = RawIdentifier(id)
-        Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), rawId, dataType))
-        rawId
+      case Some(_) => RawIdentifier(id)
     }
 
     val expr = parseExprBytes(dataType, io)
@@ -104,25 +99,23 @@ trait EveryReadIsExpression
     }
   }
 
-  def attrUserTypeParse(id: Identifier, dataType: UserType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, defEndian: Option[FixedEndian]): Unit = {
+  def attrUserTypeParse(id: Identifier, dataType: UserType, io: String, rep: RepeatSpec, defEndian: Option[FixedEndian]): Unit = {
     val newIO = dataType match {
       case knownSizeType: UserTypeFromBytes =>
         // we have a fixed buffer, thus we shall create separate IO for it
         val rawId = RawIdentifier(id)
         val byteType = knownSizeType.bytes
 
-        attrParse2(rawId, byteType, io, extraAttrs, rep, true, defEndian)
+        attrParse2(rawId, byteType, io, rep, true, defEndian)
 
         val extraType = rep match {
           case NoRepeat => byteType
           case _ => ArrayType(byteType)
         }
 
-        Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), rawId, extraType))
-
         this match {
           case thisStore: AllocateAndStoreIO =>
-            thisStore.allocateIO(rawId, rep, extraAttrs)
+            thisStore.allocateIO(rawId, rep)
           case thisLocal: AllocateIOLocalVar =>
             thisLocal.allocateIO(rawId, rep)
         }
@@ -156,7 +149,6 @@ trait EveryReadIsExpression
     on: Ast.expr,
     cases: Map[Ast.expr, DataType],
     io: String,
-    extraAttrs: ListBuffer[AttrSpec],
     rep: RepeatSpec,
     defEndian: Option[FixedEndian],
     isNullable: Boolean,
@@ -169,17 +161,17 @@ trait EveryReadIsExpression
       (dataType) => {
         if (isNullable)
           condIfSetNonNull(id)
-        attrParse2(id, dataType, io, extraAttrs, rep, false, defEndian, Some(assignType))
+        attrParse2(id, dataType, io, rep, false, defEndian, Some(assignType))
       },
       (dataType) => if (switchBytesOnlyAsRaw) {
         dataType match {
           case t: BytesType =>
-            attrParse2(RawIdentifier(id), dataType, io, extraAttrs, rep, false, defEndian, Some(assignType))
+            attrParse2(RawIdentifier(id), dataType, io, rep, false, defEndian, Some(assignType))
           case _ =>
-            attrParse2(id, dataType, io, extraAttrs, rep, false, defEndian, Some(assignType))
+            attrParse2(id, dataType, io, rep, false, defEndian, Some(assignType))
         }
       } else {
-        attrParse2(id, dataType, io, extraAttrs, rep, false, defEndian, Some(assignType))
+        attrParse2(id, dataType, io, rep, false, defEndian, Some(assignType))
       }
     )
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
index 00729fd19..5e70658a2 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/GoReads.scala
@@ -16,7 +16,6 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with SwitchOps {
     id: Identifier,
     dataType: DataType,
     io: String,
-    extraAttrs: ListBuffer[AttrSpec],
     rep: RepeatSpec,
     isRaw: Boolean,
     defEndian: Option[FixedEndian],
@@ -26,7 +25,7 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with SwitchOps {
       case FixedBytesType(c, _) =>
         attrFixedContentsParse(id, c)
       case t: UserType =>
-        attrUserTypeParse(id, t, io, extraAttrs, rep, defEndian)
+        attrUserTypeParse(id, t, io, rep, defEndian)
 //      case t: BytesType =>
 //        attrBytesTypeParse(id, t, io, extraAttrs, rep, isRaw)
 //      case SwitchType(on, cases) =>
@@ -67,25 +66,23 @@ trait GoReads extends CommonReads with ObjectOrientedLanguage with SwitchOps {
     expr
   }
 
-  def attrUserTypeParse(id: Identifier, dataType: UserType, io: String, extraAttrs: ListBuffer[AttrSpec], rep: RepeatSpec, defEndian: Option[FixedEndian]): Unit = {
+  def attrUserTypeParse(id: Identifier, dataType: UserType, io: String, rep: RepeatSpec, defEndian: Option[FixedEndian]): Unit = {
     val newIO = dataType match {
       case knownSizeType: UserTypeFromBytes =>
         // we have a fixed buffer, thus we shall create separate IO for it
         val rawId = RawIdentifier(id)
         val byteType = knownSizeType.bytes
 
-        attrParse2(rawId, byteType, io, extraAttrs, rep, true, defEndian)
+        attrParse2(rawId, byteType, io, rep, true, defEndian)
 
         val extraType = rep match {
           case NoRepeat => byteType
           case _ => ArrayType(byteType)
         }
 
-        Utils.addUniqueAttr(extraAttrs, AttrSpec(List(), rawId, extraType))
-
         this match {
           case thisStore: AllocateAndStoreIO =>
-            thisStore.allocateIO(rawId, rep, extraAttrs)
+            thisStore.allocateIO(rawId, rep)
           case thisLocal: AllocateIOLocalVar =>
             thisLocal.allocateIO(rawId, rep)
         }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index dcbe0618c..6af393a00 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -89,7 +89,7 @@ abstract class LanguageCompiler(
   def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit
   def attributeDoc(id: Identifier, doc: DocSpec): Unit = {}
 
-  def attrParse(attr: AttrLikeSpec, id: Identifier, extraAttrs: ListBuffer[AttrSpec], defEndian: Option[Endianness]): Unit
+  def attrParse(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit
   def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit
   def attrDestructor(attr: AttrLikeSpec, id: Identifier): Unit = {}
 

From cebc8f6093e848e12e7ac76f302de97e822d6b3e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 28 Oct 2018 10:32:04 +0000
Subject: [PATCH 177/230] Cleanup imports

---
 .../src/main/scala/io/kaitai/struct/languages/CppCompiler.scala | 2 --
 .../kaitai/struct/languages/components/AllocateAndStoreIO.scala | 2 --
 2 files changed, 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 911c11246..5aa060966 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -9,8 +9,6 @@ import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components._
 import io.kaitai.struct.translators.{CppTranslator, TypeDetector}
 
-import scala.collection.mutable.ListBuffer
-
 class CppCompiler(
   typeProvider: ClassTypeProvider,
   config: RuntimeConfig
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
index acd56ebe2..731cfc2ef 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/AllocateAndStoreIO.scala
@@ -3,8 +3,6 @@ package io.kaitai.struct.languages.components
 import io.kaitai.struct.datatype.DataType.{ArrayType, KaitaiStreamType}
 import io.kaitai.struct.format._
 
-import scala.collection.mutable.ListBuffer
-
 /**
   * Allocates new IO and returns attribute identifier that it will be stored
   * at. This is used for languages without garbage collection that need to

From 1f76a03efe2fc155b301dc707cec6a9acf7f6096 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 28 Oct 2018 10:32:48 +0000
Subject: [PATCH 178/230] Removed unused extra attributes created in ExtraAttrs
 for C++

---
 .../struct/languages/components/ExtraAttrs.scala    | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
index 563fa3f32..90f19d3a9 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/ExtraAttrs.scala
@@ -41,17 +41,20 @@ object ExtraAttrs {
   def forAttr(id: Identifier, dataType: DataType, condSpec: ConditionalSpec, compiler: ExtraAttrs): Iterable[AttrSpec] = {
     dataType match {
       case bt: BytesType =>
-        val rawIdAttrs = bt.process match {
+        // Byte array: only need extra attrs if `process` is used
+        bt.process match {
           case None => List()
           case Some(_) =>
             val rawId = RawIdentifier(id)
-            List(AttrSpec(List(), rawId, bt, condSpec))
+            List(AttrSpec(List(), rawId, bt, condSpec)) ++
+              compiler.extraAttrForIO(id, condSpec.repeat)
         }
-        val ioIdAttrs = compiler.extraAttrForIO(id, condSpec.repeat)
-        rawIdAttrs ++ ioIdAttrs
       case utb: UserTypeFromBytes =>
+        // User type in a substream
         val rawId = RawIdentifier(id)
-        List(AttrSpec(List(), rawId, utb.bytes, condSpec)) ++ forAttr(rawId, utb.bytes, condSpec, compiler)
+        (List(AttrSpec(List(), rawId, utb.bytes, condSpec)) ++
+          compiler.extraAttrForIO(rawId, condSpec.repeat) ++
+          forAttr(rawId, utb.bytes, condSpec, compiler)).toList.distinct
       case st: SwitchType =>
         st.cases.flatMap { case (_, caseType) =>
           forAttr(id, caseType, condSpec, compiler)

From 74699eda2e1be9518a2351afdecc1a60d92b9850 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 29 Oct 2018 11:03:38 +0000
Subject: [PATCH 179/230] CppCompiler: endif shall be at the very end of
 generated header file

---
 .../main/scala/io/kaitai/struct/languages/CppCompiler.scala | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 5aa060966..982f4474e 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -84,15 +84,15 @@ class CppCompiler(
   }
 
   override def fileFooter(topClassName: String): Unit = {
-    outHdr.puts
-    outHdr.puts(s"#endif  // ${defineName(topClassName)}")
-
     config.cppNamespace.foreach { (_) =>
       outSrc.dec
       outSrc.puts("}")
       outHdr.dec
       outHdr.puts("}")
     }
+
+    outHdr.puts
+    outHdr.puts(s"#endif  // ${defineName(topClassName)}")
   }
 
   override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = {

From 5225f5ac77baaa157f5d408d6d63f312d85305a1 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 29 Oct 2018 14:33:36 +0000
Subject: [PATCH 180/230] CppCompiler: add attribute initialization that will
 allow clean object destruction

---
 .../io/kaitai/struct/ClassCompiler.scala      | 31 +++++++++++++++++++
 .../kaitai/struct/languages/CppCompiler.scala | 10 ++++++
 .../components/LanguageCompiler.scala         |  1 +
 3 files changed, 42 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index b52cbb158..9a7163bc4 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -114,12 +114,43 @@ class ClassCompiler(
       curClass.meta.endian.contains(InheritedEndian),
       curClass.params
     )
+    compileInit(curClass)
     curClass.instances.foreach { case (instName, _) => lang.instanceClear(instName) }
     if (lang.config.autoRead)
       lang.runRead()
     lang.classConstructorFooter
   }
 
+  /**
+    * Compile initialization of class members for a given type. Typically
+    * this is only required for languages which both:
+    *
+    * * don't perform auto-initialization of object with some default
+    *   values (like 0s) on object creation,
+    * * require these members to be initialized because any other
+    *   procedures with object (e.g. destruction) will require that
+    *
+    * Currently, this is only applicable to C++ without smart pointers,
+    * as destructors we'll generate will rely on pointers being set to
+    * null.
+    * @param curClass current type to generate code for
+    */
+  def compileInit(curClass: ClassSpec) = {
+    curClass.seq.foreach((attr) => compileAttrInit(attr))
+    curClass.instances.foreach { case (_, instSpec) =>
+      instSpec match {
+        case pis: ParseInstanceSpec => compileAttrInit(pis)
+        case _: ValueInstanceSpec => // ignore for now
+      }
+    }
+  }
+
+  def compileAttrInit(originalAttr: AttrLikeSpec): Unit = {
+    val extraAttrs = ExtraAttrs.forAttr(originalAttr, lang)
+    val allAttrs = List(originalAttr) ++ extraAttrs
+    allAttrs.foreach((attr) => lang.attrInit(attr))
+  }
+
   /**
     * Compiles destructor for a given type. It should clean up everything
     * (i.e. every applicable allocated seq / instance attribute variables, and
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 982f4474e..b7ad2310e 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -289,6 +289,16 @@ class CppCompiler(
     outHdr.puts( " */")
   }
 
+  override def attrInit(attr: AttrLikeSpec): Unit = {
+    attr.dataTypeComposite match {
+      case _: UserType | _: ArrayType | KaitaiStreamType =>
+        // data type will be pointer to user type, std::vector or stream, so we need to init it
+        outSrc.puts(s"${privateMemberName(attr.id)} = 0;")
+      case _ =>
+        // no init required for value types
+    }
+  }
+
   override def attrDestructor(attr: AttrLikeSpec, id: Identifier): Unit = {
     val checkLazy = if (attr.isLazy) {
       Some(calculatedFlagForName(id))
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index 6af393a00..922e94294 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -91,6 +91,7 @@ abstract class LanguageCompiler(
 
   def attrParse(attr: AttrLikeSpec, id: Identifier, defEndian: Option[Endianness]): Unit
   def attrParseHybrid(leProc: () => Unit, beProc: () => Unit): Unit
+  def attrInit(attr: AttrLikeSpec): Unit = {}
   def attrDestructor(attr: AttrLikeSpec, id: Identifier): Unit = {}
 
   def attrFixedContentsParse(attrName: Identifier, contents: Array[Byte]): Unit

From ef8019a09ff64556ae47542c4026948e913bf8ec Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 2 Nov 2018 09:10:52 +0000
Subject: [PATCH 181/230] Pumped sbt-native-packager to 1.3.12, version that
 fixes problems with Java 10:
 https://github.com/sbt/sbt-native-packager/issues/1145

---
 project/plugins.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/plugins.sbt b/project/plugins.sbt
index 322ef8094..5973f265a 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,5 +1,5 @@
 logLevel := Level.Warn
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.2")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.12")
 addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.7.0")
 addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.21")

From 3296b328c26844048fe1e1e1033d0050b269e5dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rafa=C5=82=20Harabie=C5=84?= <rafalh92@outlook.com>
Date: Mon, 5 Nov 2018 00:39:00 +0100
Subject: [PATCH 182/230] Fix invalid Java code generation when using
 repeat-expr with optional u4 field

---
 .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 4eee672e5..e74c581f7 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -371,8 +371,8 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def condRepeatExprHeader(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, repeatExpr: expr): Unit = {
     if (needRaw)
-      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>((int) (${expression(repeatExpr)}));")
-    out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}((int) (${expression(repeatExpr)}));")
+      out.puts(s"${privateMemberName(RawIdentifier(id))} = new ArrayList<byte[]>(((Number) (${expression(repeatExpr)})).intValue());")
+    out.puts(s"${idToStr(id)} = new ${kaitaiType2JavaType(ArrayType(dataType))}(((Number) (${expression(repeatExpr)})).intValue());")
     out.puts(s"for (int i = 0; i < ${expression(repeatExpr)}; i++) {")
     out.inc
 

From 4e5cab8adadd118a92cb44c3f6b16f4c96aa1b72 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 5 Nov 2018 13:38:32 +0000
Subject: [PATCH 183/230] TypeValidator: added better validation of expressions
 in parse instances

---
 .../struct/precompile/TypeValidator.scala     | 61 ++++++++++++++++++-
 1 file changed, 60 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala
index 6bd2b1210..94d1a504f 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/TypeValidator.scala
@@ -43,7 +43,7 @@ class TypeValidator(specs: ClassSpecs, topClass: ClassSpec) {
     curClass.instances.foreach { case (_, inst) =>
       inst match {
         case pis: ParseInstanceSpec =>
-          validateAttr(pis)
+          validateParseInstance(pis)
         case vis: ValueInstanceSpec =>
           // TODO
       }
@@ -77,6 +77,22 @@ class TypeValidator(specs: ClassSpecs, topClass: ClassSpec) {
     validateDataType(attr.dataType, path)
   }
 
+  def validateParseInstance(pis: ParseInstanceSpec): Unit = {
+    validateAttr(pis)
+
+    Log.typeValid.info(() => s"validateParseInstance(${pis.id.humanReadable})")
+
+    pis.io match {
+      case Some(io) => checkAssertObject(io, KaitaiStreamType, "IO stream", pis.path, "io")
+      case None => // all good
+    }
+
+    pis.pos match {
+      case Some(pos) => checkAssert[IntType](pos, "integer", pis.path, "pos")
+      case None => // all good
+    }
+  }
+
   /**
     * Validates single non-composite data type, checking all expressions
     * inside data type definition.
@@ -189,4 +205,47 @@ class TypeValidator(specs: ClassSpecs, topClass: ClassSpec) {
         throw new ErrorInInput(err, path ++ List(pathKey))
     }
   }
+
+  /**
+    * Checks that expression's type conforms to a given datatype, otherwise
+    * throw a human-readable exception, with some pointers that would help
+    * finding the expression in source .ksy.
+    *
+    * This version works with case objects.
+    *
+    * @param expr expression to check
+    * @param expectStr string to include
+    * @param path path to expression base
+    * @param pathKey key that contains expression in given path
+    */
+  def checkAssertObject(
+    expr: Ast.expr,
+    expected: Object,
+    expectStr: String,
+    path: List[String],
+    pathKey: String
+  ): Unit = {
+    try {
+      val detected = detector.detectType(expr)
+      if (detected == expected) {
+        // good
+      } else {
+        detected match {
+          case st: SwitchType =>
+            val combinedType = st.combinedType
+            if (combinedType == expected) {
+              // good
+            } else {
+              throw YAMLParseException.exprType(expectStr, combinedType, path ++ List(pathKey))
+            }
+          case actual => throw YAMLParseException.exprType(expectStr, actual, path ++ List(pathKey))
+        }
+      }
+    } catch {
+      case err: InvalidIdentifier =>
+        throw new ErrorInInput(err, path ++ List(pathKey))
+      case err: ExpressionError =>
+        throw new ErrorInInput(err, path ++ List(pathKey))
+    }
+  }
 }

From 8fe1ab53f391f2fe9dcd23b524fb1e92bbefa08e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 5 Nov 2018 15:00:30 +0000
Subject: [PATCH 184/230] Utils: added safeLookup

---
 .../main/scala/io/kaitai/struct/Utils.scala   | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/Utils.scala b/shared/src/main/scala/io/kaitai/struct/Utils.scala
index f32c50e2f..bae0c7050 100644
--- a/shared/src/main/scala/io/kaitai/struct/Utils.scala
+++ b/shared/src/main/scala/io/kaitai/struct/Utils.scala
@@ -113,4 +113,23 @@ object Utils {
     } else {
       fullPath
     }
+
+  /**
+    * Performs safe lookup for up to `len` character in a given
+    * string `src`, starting at `from`.
+    * @param src string to work on
+    * @param from starting character index
+    * @param len max length of substring
+    * @return substring of `src`, starting at `from`, up to `len` chars max
+    */
+  def safeLookup(src: String, from: Int, len: Int): String = {
+    val maxLen = src.length
+    if (from >= maxLen) {
+      ""
+    } else {
+      val to = from + len
+      val safeTo = if (to > maxLen) maxLen else to
+      src.substring(from, safeTo)
+    }
+  }
 }

From 0feac5da14302e3ca4b01ec764293e449edeff94 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 5 Nov 2018 15:22:53 +0000
Subject: [PATCH 185/230] Added `get*ValueExpression` to get expression
 straight away from a YAML Map

---
 .../io/kaitai/struct/format/ParseUtils.scala  | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala b/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala
index 614f6954d..83430cdb1 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/ParseUtils.scala
@@ -1,6 +1,7 @@
 package io.kaitai.struct.format
 
 import io.kaitai.struct.Utils
+import io.kaitai.struct.exprlang.{Ast, Expressions}
 
 object ParseUtils {
   def ensureLegalKeys(src: Map[String, Any], legalKeys: Set[String], path: List[String], where: Option[String] = None) = {
@@ -83,6 +84,24 @@ object ParseUtils {
     }
   }
 
+  def getValueExpression(src: Map[String, Any], field: String, path: List[String]): Ast.expr = {
+    try {
+      Expressions.parse(getValueStr(src, field, path))
+    } catch {
+      case epe: Expressions.ParseException =>
+        throw YAMLParseException.expression(epe, path)
+    }
+  }
+
+  def getOptValueExpression(src: Map[String, Any], field: String, path: List[String]): Option[Ast.expr] = {
+    try {
+      getOptValueStr(src, field, path).map(Expressions.parse)
+    } catch {
+      case epe: Expressions.ParseException =>
+        throw YAMLParseException.expression(epe, path)
+    }
+  }
+
   /**
     * Gets a list of T-typed values from a given YAML map's key "field",
     * reporting errors accurately and ensuring type safety.

From 7159ab880ed83cfc84b01b1ab59321a5b8056d07 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 5 Nov 2018 15:23:46 +0000
Subject: [PATCH 186/230] Slightly more user friendly expression parsing error
 message with a suggestion about typical errors

---
 .../struct/format/YAMLParseException.scala      | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala b/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala
index bedf3bb70..a38638fbf 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala
@@ -1,6 +1,7 @@
 package io.kaitai.struct.format
 
 import fastparse.StringReprOps
+import io.kaitai.struct.Utils
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.exprlang.Expressions
 
@@ -37,8 +38,22 @@ object YAMLParseException {
   def expression(epe: Expressions.ParseException, path: List[String]): YAMLParseException = {
     val f = epe.failure
     val pos = StringReprOps.prettyIndex(f.extra.input, f.index)
+
+    // Try to diagnose most common errors and provide a friendly suggestion
+    val lookup2 = Utils.safeLookup(epe.src, f.index, 2)
+    val suggesion: String = (if (lookup2 == "&&") {
+      Some("and")
+    } else if (lookup2 == "||") {
+      Some("or")
+    } else {
+      None
+    }).map((x) => s", did you mean '$x'?").getOrElse("")
+
+    f.extra.traced.expected
+
     new YAMLParseException(
-      s"parsing expression '${epe.src}' failed on $pos, expected ${f.extra.traced.expected.replaceAll("\n", "\\n")}",
+      s"parsing expression '${epe.src}' failed on $pos, " +
+        s"expected ${f.extra.traced.expected.replaceAll("\n", "\\n")}$suggesion",
       path
     )
   }

From 566b980b04af40fb09496edc6352cd14bba2339f Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 5 Nov 2018 15:26:06 +0000
Subject: [PATCH 187/230] Better error messages + automatic exception catch on
 parsing of expressions

---
 .../io/kaitai/struct/format/AttrSpec.scala    | 19 ++++-------
 .../kaitai/struct/format/InstanceSpec.scala   |  8 ++---
 .../io/kaitai/struct/format/ProcessExpr.scala | 33 +++++++++++--------
 3 files changed, 29 insertions(+), 31 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
index 444d81578..613898f06 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
@@ -173,20 +173,20 @@ object AttrSpec {
     val process = ProcessExpr.fromStr(ParseUtils.getOptValueStr(srcMap, "process", path), path)
     // TODO: add proper path propagation
     val contents = srcMap.get("contents").map(parseContentSpec(_, path ++ List("contents")))
-    val size = ParseUtils.getOptValueStr(srcMap, "size", path).map(Expressions.parse)
+    val size = ParseUtils.getOptValueExpression(srcMap, "size", path)
     val sizeEos = ParseUtils.getOptValueBool(srcMap, "size-eos", path).getOrElse(false)
-    val ifExpr = ParseUtils.getOptValueStr(srcMap, "if", path).map(Expressions.parse)
+    val ifExpr = ParseUtils.getOptValueExpression(srcMap, "if", path)
     val encoding = ParseUtils.getOptValueStr(srcMap, "encoding", path)
     val repeat = ParseUtils.getOptValueStr(srcMap, "repeat", path)
-    val repeatExpr = ParseUtils.getOptValueStr(srcMap, "repeat-expr", path).map(Expressions.parse)
-    val repeatUntil = ParseUtils.getOptValueStr(srcMap, "repeat-until", path).map(Expressions.parse)
+    val repeatExpr = ParseUtils.getOptValueExpression(srcMap, "repeat-expr", path)
+    val repeatUntil = ParseUtils.getOptValueExpression(srcMap, "repeat-until", path)
     val terminator = ParseUtils.getOptValueInt(srcMap, "terminator", path)
     val consume = ParseUtils.getOptValueBool(srcMap, "consume", path).getOrElse(true)
     val include = ParseUtils.getOptValueBool(srcMap, "include", path).getOrElse(false)
     val eosError = ParseUtils.getOptValueBool(srcMap, "eos-error", path).getOrElse(true)
     val padRight = ParseUtils.getOptValueInt(srcMap, "pad-right", path)
     val enum = ParseUtils.getOptValueStr(srcMap, "enum", path)
-    val parent = ParseUtils.getOptValueStr(srcMap, "parent", path).map(Expressions.parse)
+    val parent = ParseUtils.getOptValueExpression(srcMap, "parent", path)
 
     val typObj = srcMap.get("type")
 
@@ -265,18 +265,11 @@ object AttrSpec {
     metaDef: MetaSpec,
     arg: YamlAttrArgs
   ): DataType = {
-    val _on = ParseUtils.getValueStr(switchSpec, "switch-on", path)
+    val on = ParseUtils.getValueExpression(switchSpec, "switch-on", path)
     val _cases = ParseUtils.getValueMapStrStr(switchSpec, "cases", path)
 
     ParseUtils.ensureLegalKeys(switchSpec, LEGAL_KEYS_SWITCH, path)
 
-    val on = try {
-      Expressions.parse(_on)
-    } catch {
-      case epe: Expressions.ParseException =>
-        throw YAMLParseException.expression(epe, path ++ List("switch-on"))
-    }
-
     val cases = _cases.map { case (condition, typeName) =>
       val casePath = path ++ List("cases", condition)
       val condType = DataType.fromYaml(
diff --git a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
index 6d6ca2821..2577ac99d 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
@@ -41,7 +41,7 @@ object InstanceSpec {
   def fromYaml(src: Any, path: List[String], metaDef: MetaSpec, id: InstanceIdentifier): InstanceSpec = {
     val srcMap = ParseUtils.asMapStr(src, path)
 
-    ParseUtils.getOptValueStr(srcMap, "value", path).map(Expressions.parse) match {
+    ParseUtils.getOptValueExpression(srcMap, "value", path) match {
       case Some(value) =>
         // value instance
         ParseUtils.ensureLegalKeys(srcMap, LEGAL_KEYS_VALUE_INST, path, Some("value instance"))
@@ -54,7 +54,7 @@ object InstanceSpec {
             Ast.expr.EnumById(Ast.identifier(enumName), value)
         }
 
-        val ifExpr = ParseUtils.getOptValueStr(srcMap, "if", path).map(Expressions.parse)
+        val ifExpr = ParseUtils.getOptValueExpression(srcMap, "if", path)
 
         ValueInstanceSpec(
           path,
@@ -65,8 +65,8 @@ object InstanceSpec {
         )
       case None =>
         // normal positional instance
-        val pos = ParseUtils.getOptValueStr(srcMap, "pos", path).map(Expressions.parse)
-        val io = ParseUtils.getOptValueStr(srcMap, "io", path).map(Expressions.parse)
+        val pos = ParseUtils.getOptValueExpression(srcMap, "pos", path)
+        val io = ParseUtils.getOptValueExpression(srcMap, "io", path)
 
         val fakeAttrMap = srcMap.filterKeys((key) => key != "pos" && key != "io")
         val a = AttrSpec.fromYaml(fakeAttrMap, path, metaDef, id)
diff --git a/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala b/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
index 7fe8a6cf9..9764f8731 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/ProcessExpr.scala
@@ -20,20 +20,25 @@ object ProcessExpr {
       case None =>
         None
       case Some(op) =>
-        Some(op match {
-          case "zlib" =>
-            ProcessZlib
-          case ReXor(arg) =>
-            ProcessXor(Expressions.parse(arg))
-          case ReRotate(dir, arg) =>
-            ProcessRotate(dir == "l", Expressions.parse(arg))
-          case ReCustom(name, args) =>
-            ProcessCustom(name.split('.').toList, Expressions.parseList(args))
-          case ReCustomNoArg(name) =>
-            ProcessCustom(name.split('.').toList, Seq())
-          case _ =>
-            throw YAMLParseException.badProcess(op, path)
-        })
+        try {
+          Some(op match {
+            case "zlib" =>
+              ProcessZlib
+            case ReXor(arg) =>
+              ProcessXor(Expressions.parse(arg))
+            case ReRotate(dir, arg) =>
+              ProcessRotate(dir == "l", Expressions.parse(arg))
+            case ReCustom(name, args) =>
+              ProcessCustom(name.split('.').toList, Expressions.parseList(args))
+            case ReCustomNoArg(name) =>
+              ProcessCustom(name.split('.').toList, Seq())
+            case _ =>
+              throw YAMLParseException.badProcess(op, path)
+          })
+        } catch {
+          case epe: Expressions.ParseException =>
+            throw YAMLParseException.expression(epe, path)
+        }
     }
   }
 }

From 302e82d7ba9b73f35eb946896c3d9e2eb95a2c7d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 6 Nov 2018 18:31:23 +0000
Subject: [PATCH 188/230] Added zip publisher

---
 lib_bintray.sh                                | 19 +++++++++++++++++++
 ...to_bintray.sh => publish_deb_to_bintray.sh |  0
 publish_zip_to_bintray.sh                     | 17 +++++++++++++++++
 3 files changed, 36 insertions(+)
 rename publish_to_bintray.sh => publish_deb_to_bintray.sh (100%)
 create mode 100644 publish_zip_to_bintray.sh

diff --git a/lib_bintray.sh b/lib_bintray.sh
index 6d3729ac8..6ccc6faee 100644
--- a/lib_bintray.sh
+++ b/lib_bintray.sh
@@ -31,6 +31,25 @@ bintray_create_version()
 #         --data "{ \"name\": \"$version\", \"release_notes\": \"auto\", \"release_url\": \"$BASE_DESC/$RPM_NAME\", \"released\": \"\" }"
 }
 
+##
+# Uploads generic file to Bintray.
+#
+# Input:
+# $1 = filename to upload
+bintray_upload_generic()
+{
+	local filename="$1"
+
+	echo "bintray_upload_generic(repo=${BINTRAY_REPO}, package=${BINTRAY_PACKAGE}, version=${BINTRAY_VERSION}, filename=${filename})"
+
+	curl $BINTRAY_CURL_ARGS -f \
+		-T "$filename" \
+		"-u$BINTRAY_USER:$BINTRAY_API_KEY" \
+		-H "X-Bintray-Package: $BINTRAY_PACKAGE" \
+		-H "X-Bintray-Version: $BINTRAY_VERSION" \
+		https://api.bintray.com/content/$BINTRAY_ACCOUNT/$BINTRAY_REPO/
+}
+
 ##
 # Uploads deb package to Bintray.
 #
diff --git a/publish_to_bintray.sh b/publish_deb_to_bintray.sh
similarity index 100%
rename from publish_to_bintray.sh
rename to publish_deb_to_bintray.sh
diff --git a/publish_zip_to_bintray.sh b/publish_zip_to_bintray.sh
new file mode 100644
index 000000000..a21705393
--- /dev/null
+++ b/publish_zip_to_bintray.sh
@@ -0,0 +1,17 @@
+#!/bin/sh -ef
+
+. ./lib_bintray.sh
+
+# Config
+BINTRAY_USER=greycat
+BINTRAY_ACCOUNT=kaitai-io
+BINTRAY_REPO=universal_unstable
+BINTRAY_PACKAGE=kaitai-struct-compiler
+BINTRAY_VERSION="$KAITAI_STRUCT_VERSION"
+# BINTRAY_API_KEY comes from encrypted variables from web UI
+
+#BINTRAY_CURL_ARGS=-v
+
+bintray_create_version
+bintray_upload_generic "jvm/target/universal/kaitai-struct-compiler-${KAITAI_STRUCT_VERSION}.zip"
+bintray_publish_version

From 66c582dd948f7573a0622f9a4ac590da4c974833 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 6 Nov 2018 18:44:11 +0000
Subject: [PATCH 189/230] Enabled executable bit

---
 publish_zip_to_bintray.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 publish_zip_to_bintray.sh

diff --git a/publish_zip_to_bintray.sh b/publish_zip_to_bintray.sh
old mode 100644
new mode 100755

From 06c5dfd0934e37883784ee53bbcceef1dc51a0c6 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 14 Nov 2018 09:34:38 +0000
Subject: [PATCH 190/230] Added C++ 98/11 support option

---
 .../scala/io/kaitai/struct/JavaMain.scala     | 47 ++++++++++++++++---
 .../io/kaitai/struct/RuntimeConfig.scala      | 40 +++++++++++++++-
 .../kaitai/struct/languages/CppCompiler.scala |  4 +-
 3 files changed, 81 insertions(+), 10 deletions(-)

diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
index 7dea6a958..d41e05838 100644
--- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
+++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
@@ -7,6 +7,7 @@ import io.kaitai.struct.CompileLog._
 import io.kaitai.struct.JavaMain.CLIConfig
 import io.kaitai.struct.format.{ClassSpec, ClassSpecs, KSVersion, YAMLParseException}
 import io.kaitai.struct.formats.JavaKSYParser
+import io.kaitai.struct.languages.CppCompiler
 import io.kaitai.struct.languages.components.LanguageCompilerStatic
 import io.kaitai.struct.precompile.ErrorInInput
 
@@ -24,8 +25,9 @@ object JavaMain {
     runtime: RuntimeConfig = RuntimeConfig()
   )
 
-  val ALL_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet
-  val VALID_LANGS = ALL_LANGS + "all"
+  val ALL_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet - "cpp_stl" + "cpp_stl_98" + "cpp_stl_11"
+  val VALID_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet + "all"
+  val CPP_STANDARDS = Set("98", "11")
 
   def parseCommandLine(args: Array[String]): Option[CLIConfig] = {
     val parser = new scopt.OptionParser[CLIConfig](BuildInfo.name) {
@@ -65,9 +67,32 @@ object JavaMain {
       } text(".ksy library search path(s) for imports (see also KSPATH env variable)")
 
       opt[String]("cpp-namespace") valueName("<namespace>") action { (x, c) =>
-        c.copy(runtime = c.runtime.copy(cppNamespace = x.split("::").toList))
+        c.copy(
+          runtime = c.runtime.copy(
+            cppConfig = c.runtime.cppConfig.copy(
+              namespace = x.split("::").toList
+            )
+          )
+        )
       } text("C++ namespace (C++ only, default: none)")
 
+      opt[String]("cpp-standard") valueName("<standard>") action { (x, c) =>
+        c.copy(
+          runtime = c.runtime.copy(
+            cppConfig = x match {
+              case "98" => c.runtime.cppConfig.copyAsCpp98()
+              case "11" => c.runtime.cppConfig.copyAsCpp11()
+            }
+          )
+        )
+      } text("C++ standard to target (C++ only, supported: 98, 11, default: 98)") validate { x =>
+        if (CPP_STANDARDS.contains(x)) {
+          success
+        } else {
+          failure(s"'$x' is not a valid C++ standard to target; valid ones are: ${CPP_STANDARDS.mkString(", ")}")
+        }
+      }
+
       opt[String]("go-package") valueName("<package>") action { (x, c) =>
         c.copy(runtime = c.runtime.copy(goPackage = x))
       } text("Go package (Go only, default: none)")
@@ -272,10 +297,19 @@ class JavaMain(config: CLIConfig) {
 
   def compileOneLang(specs: ClassSpecs, langStr: String, outDir: String): Map[String, SpecEntry] = {
     Log.fileOps.info(() => s"... compiling it for $langStr... ")
-    val lang = LanguageCompilerStatic.byString(langStr)
+
+    val (lang, fixedRuntime) = langStr match {
+      case "cpp_stl_98" =>
+        (CppCompiler, config.runtime.copy(cppConfig = config.runtime.cppConfig.copyAsCpp98()))
+      case "cpp_stl_11" =>
+        (CppCompiler, config.runtime.copy(cppConfig = config.runtime.cppConfig.copyAsCpp11()))
+      case _ =>
+        (LanguageCompilerStatic.byString(langStr), config.runtime)
+    }
+
     specs.map { case (_, classSpec) =>
       val res = try {
-        compileSpecAndWriteToFile(specs, classSpec, lang, outDir)
+        compileSpecAndWriteToFile(specs, classSpec, lang, fixedRuntime, outDir)
       } catch {
         case ex: Throwable =>
           if (config.throwExceptions)
@@ -290,9 +324,10 @@ class JavaMain(config: CLIConfig) {
     specs: ClassSpecs,
     spec: ClassSpec,
     lang: LanguageCompilerStatic,
+    runtime: RuntimeConfig,
     outDir: String
   ): SpecSuccess = {
-    val res = Main.compile(specs, spec, lang, config.runtime)
+    val res = Main.compile(specs, spec, lang, runtime)
     res.files.foreach { (file) =>
       Log.fileOps.info(() => s".... writing ${file.fileName}")
 
diff --git a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
index d16f00840..9629e58bc 100644
--- a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
@@ -1,5 +1,41 @@
 package io.kaitai.struct
 
+/**
+  * C++-specific runtime configuration of the compiler.
+  * @param usePragmaOnce If true, use `#pragma once` in headers. If false (default),
+  *                      use `#ifndef`-`#define`-`#endif` guards.
+  * @param pointers Choose which style of pointers to use.
+  */
+case class CppRuntimeConfig(
+  namespace: List[String] = List(),
+  usePragmaOnce: Boolean = false,
+  pointers: CppRuntimeConfig.Pointers = CppRuntimeConfig.RawPointers
+) {
+  /**
+    * Copies this C++ runtime config, applying all the default settings for
+    * C++98 target.
+    */
+  def copyAsCpp98() = copy(
+    usePragmaOnce = false,
+    pointers = CppRuntimeConfig.RawPointers
+  )
+
+  /**
+    * Copies this C++ runtime config, applying all the default settings for
+    * C++11 target.
+    */
+  def copyAsCpp11() = copy(
+    usePragmaOnce = true,
+    pointers = CppRuntimeConfig.SharedPointers
+  )
+}
+
+object CppRuntimeConfig {
+  trait Pointers
+  case object RawPointers extends Pointers
+  case object SharedPointers extends Pointers
+}
+
 /**
   * Runtime configuration of the compiler which controls certain aspects of
   * code generation for target languages.
@@ -17,7 +53,7 @@ package io.kaitai.struct
   *                    "opaque" type, i.e. an external KaitaiStruct-compatible type
   *                    defined somewhere else. If false, it will be reported as
   *                    precompile error.
-  * @param cppNamespace C++ namespace
+  * @param cppConfig C++-specific configuration
   * @param goPackage Go package name
   * @param javaPackage Java package name
   * @param javaFromFileClass Java class to be invoked in `fromFile` helper methods
@@ -29,7 +65,7 @@ case class RuntimeConfig(
   autoRead: Boolean = true,
   readStoresPos: Boolean = false,
   opaqueTypes: Boolean = false,
-  cppNamespace: List[String] = List(),
+  cppConfig: CppRuntimeConfig = CppRuntimeConfig(),
   goPackage: String = "",
   javaPackage: String = "",
   javaFromFileClass: String = "io.kaitai.struct.ByteBufferKaitaiStream",
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index b7ad2310e..764e184b8 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -75,7 +75,7 @@ class CppCompiler(
     )
     outHdr.puts("#endif")
 
-    config.cppNamespace.foreach { (namespace) =>
+    config.cppConfig.namespace.foreach { (namespace) =>
       outSrc.puts(s"namespace $namespace {")
       outSrc.inc
       outHdr.puts(s"namespace $namespace {")
@@ -84,7 +84,7 @@ class CppCompiler(
   }
 
   override def fileFooter(topClassName: String): Unit = {
-    config.cppNamespace.foreach { (_) =>
+    config.cppConfig.namespace.foreach { (_) =>
       outSrc.dec
       outSrc.puts("}")
       outHdr.dec

From 25fb1eee61d07bdc8b199445776836ce9a6606ef Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 16 Nov 2018 22:00:34 +0000
Subject: [PATCH 191/230] CppCompiler: implemented usePragmaOnce config option

---
 .../io/kaitai/struct/languages/CppCompiler.scala   | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 764e184b8..0e5bf3c31 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -55,8 +55,12 @@ class CppCompiler(
     outSrcHeader.puts("#include \"" + outFileName(topClassName) + ".h\"")
     outSrcHeader.puts
 
-    outHdrHeader.puts(s"#ifndef ${defineName(topClassName)}")
-    outHdrHeader.puts(s"#define ${defineName(topClassName)}")
+    if (config.cppConfig.usePragmaOnce) {
+      outHdrHeader.puts("#pragma once")
+    } else {
+      outHdrHeader.puts(s"#ifndef ${defineName(topClassName)}")
+      outHdrHeader.puts(s"#define ${defineName(topClassName)}")
+    }
     outHdrHeader.puts
     outHdrHeader.puts(s"// $headerComment")
     outHdrHeader.puts
@@ -91,8 +95,10 @@ class CppCompiler(
       outHdr.puts("}")
     }
 
-    outHdr.puts
-    outHdr.puts(s"#endif  // ${defineName(topClassName)}")
+    if (!config.cppConfig.usePragmaOnce) {
+      outHdr.puts
+      outHdr.puts(s"#endif  // ${defineName(topClassName)}")
+    }
   }
 
   override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = {

From cc373b2283e65810a315ec7488a6f288bb002f6e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 7 Dec 2018 00:57:49 +0000
Subject: [PATCH 192/230] Fixed typo

---
 .../scala/io/kaitai/struct/format/YAMLParseException.scala    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala b/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala
index a38638fbf..bf5829e00 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/YAMLParseException.scala
@@ -41,7 +41,7 @@ object YAMLParseException {
 
     // Try to diagnose most common errors and provide a friendly suggestion
     val lookup2 = Utils.safeLookup(epe.src, f.index, 2)
-    val suggesion: String = (if (lookup2 == "&&") {
+    val suggestion: String = (if (lookup2 == "&&") {
       Some("and")
     } else if (lookup2 == "||") {
       Some("or")
@@ -53,7 +53,7 @@ object YAMLParseException {
 
     new YAMLParseException(
       s"parsing expression '${epe.src}' failed on $pos, " +
-        s"expected ${f.extra.traced.expected.replaceAll("\n", "\\n")}$suggesion",
+        s"expected ${f.extra.traced.expected.replaceAll("\n", "\\n")}$suggestion",
       path
     )
   }

From 42cb8e40e729f422a636eb77200d389f4535d8c4 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 17 Dec 2018 20:01:17 +0000
Subject: [PATCH 193/230] Enforce UTF-8 encoding on file reading to avoid
 encoding and BOM handling problems

---
 .../io/kaitai/struct/formats/JavaKSYParser.scala  | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala b/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala
index 57e371564..b6326b20e 100644
--- a/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala
+++ b/jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala
@@ -1,6 +1,7 @@
 package io.kaitai.struct.formats
 
-import java.io.{File, FileReader}
+import java.io._
+import java.nio.charset.Charset
 import java.util.{List => JList, Map => JMap}
 
 import io.kaitai.struct.JavaMain.CLIConfig
@@ -25,11 +26,19 @@ object JavaKSYParser {
 
   def fileNameToSpec(yamlFilename: String): ClassSpec = {
     Log.fileOps.info(() => s"reading $yamlFilename...")
-    val scalaSrc = readerToYaml(new FileReader(yamlFilename))
+
+    // This complex string of classes is due to the fact that Java's
+    // default "FileReader" implementation always uses system locale,
+    // which screws up encoding on some systems and screws up reading
+    // UTF-8 files with BOM
+    val fis = new FileInputStream(yamlFilename)
+    val isr = new InputStreamReader(fis, Charset.forName("UTF-8"))
+    val br = new BufferedReader(isr)
+    val scalaSrc = readerToYaml(br)
     ClassSpec.fromYaml(scalaSrc)
   }
 
-  def readerToYaml(reader: FileReader): Any = {
+  def readerToYaml(reader: Reader): Any = {
     val yamlLoader = new Yaml(new SafeConstructor)
     val javaSrc = yamlLoader.load(reader)
     yamlJavaToScala(javaSrc)

From 3afe5fadf9af79d8308091a8df0c65677a43f52d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 18 Dec 2018 11:26:19 +0000
Subject: [PATCH 194/230] * CppRuntimeConfig: added UniqueAndRawPointers and
 use it by default   in C++11 mode. * CppCompiler: started to add code
 generation for SharedPointers and   UniqueAndRawPointers modes.   * Removed
 most of destructor generation in SharedPointers mode.   * Added `nullPtr`
 helper that generates relevant null pointer.   * Added `nonOwningPointer`
 helper that generates expression that     converts existing owning pointer to
 non-owning pointer, used in     seq / instance attribute getters.   *
 `kaitaiToNativeType` now requires config for invocation * CppTranslator:
 added config requirement, generate static casts   differently based on
 pointer mode chosen. * DataType: added concept of "owning" / "non-owning"
 datatype.   * ComplexType: common ancestor for UserType + KaitaiStructType.  
 * UserType: added CalcUserType, to represent non-owning user type    
 (typically, result of calculation).   * KaitaiStructType: added
 CalcKaitaiStructType, to represent     non-owning generic struct type.   *
 Added relevant methods to check owning status (`isOwning`) and     helper
 method to get non-owning variant of current type     (`asNonOwning`). *
 ClassSpec, ValueTypesDeriver: generate non-owning types, where   applicable.
 * CppTranslator: added configuration passing, as translator results   now
 depend on chosen pointer model. * all `languages/`: added dataType argument
 to   `instanceCheckCacheAndReturn` and `instanceReturn`. * ClassCompiler:
 call theses new methods, passing relevant dataType.

---
 .../struct/translators/TranslatorSpec.scala   |   2 +-
 .../io/kaitai/struct/ClassCompiler.scala      |   6 +-
 .../io/kaitai/struct/ClassTypeProvider.scala  |   2 +-
 .../io/kaitai/struct/GoClassCompiler.scala    |   4 +-
 .../io/kaitai/struct/RuntimeConfig.scala      |   5 +-
 .../io/kaitai/struct/RustClassCompiler.scala  |   4 +-
 .../io/kaitai/struct/datatype/DataType.scala  |  68 +++++++-
 .../io/kaitai/struct/format/ClassSpec.scala   |   7 +-
 .../struct/languages/CSharpCompiler.scala     |   8 +-
 .../kaitai/struct/languages/CppCompiler.scala | 160 +++++++++++++-----
 .../kaitai/struct/languages/GoCompiler.scala  |   8 +-
 .../struct/languages/JavaCompiler.scala       |   8 +-
 .../struct/languages/JavaScriptCompiler.scala |   6 +-
 .../kaitai/struct/languages/LuaCompiler.scala |   6 +-
 .../kaitai/struct/languages/PHPCompiler.scala |   8 +-
 .../struct/languages/PerlCompiler.scala       |   4 +-
 .../struct/languages/PythonCompiler.scala     |   6 +-
 .../struct/languages/RubyCompiler.scala       |   4 +-
 .../struct/languages/RustCompiler.scala       |   6 +-
 .../components/LanguageCompiler.scala         |   4 +-
 .../struct/precompile/ValueTypesDeriver.scala |   2 +-
 .../struct/translators/CppTranslator.scala    |  19 ++-
 22 files changed, 251 insertions(+), 96 deletions(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
index ff898d0a0..c3d9060e5 100644
--- a/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TranslatorSpec.scala
@@ -578,7 +578,7 @@ class TranslatorSpec extends FunSuite {
     val goOutput = new StringLanguageOutputWriter("  ")
 
     val langs = Map[LanguageCompilerStatic, AbstractTranslator with TypeDetector](
-      CppCompiler -> new CppTranslator(tp, new ImportList()),
+      CppCompiler -> new CppTranslator(tp, new ImportList(), RuntimeConfig()),
       CSharpCompiler -> new CSharpTranslator(tp, new ImportList()),
       GoCompiler -> new GoTranslator(goOutput, tp, new ImportList()),
       JavaCompiler -> new JavaTranslator(tp, new ImportList()),
diff --git a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
index 9a7163bc4..b05eff43c 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassCompiler.scala
@@ -81,7 +81,7 @@ class ClassCompiler(
       curClass.seq ++
       curClass.params ++
       List(
-        AttrSpec(List(), RootIdentifier, UserTypeInstream(topClassName, None)),
+        AttrSpec(List(), RootIdentifier, CalcUserType(topClassName, None)),
         AttrSpec(List(), ParentIdentifier, curClass.parentType)
       ) ++
       ExtraAttrs.forClassSpec(curClass, lang)
@@ -313,7 +313,7 @@ class ClassCompiler(
     lang.instanceHeader(className, instName, dataType, instSpec.isNullable)
     if (lang.innerDocstrings)
       compileInstanceDoc(instName, instSpec)
-    lang.instanceCheckCacheAndReturn(instName)
+    lang.instanceCheckCacheAndReturn(instName, dataType)
 
     instSpec match {
       case vi: ValueInstanceSpec =>
@@ -325,7 +325,7 @@ class ClassCompiler(
     }
 
     lang.instanceSetCalculated(instName)
-    lang.instanceReturn(instName)
+    lang.instanceReturn(instName, dataType)
     lang.instanceFooter
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
index 9f36a0f7f..04e8258da 100644
--- a/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
+++ b/shared/src/main/scala/io/kaitai/struct/ClassTypeProvider.scala
@@ -63,7 +63,7 @@ class ClassTypeProvider(classSpecs: ClassSpecs, var topClass: ClassSpec) extends
       case GenericStructClassSpec =>
         KaitaiStructType
       case cs: ClassSpec =>
-        val ut = UserTypeInstream(cs.name, None)
+        val ut = CalcUserType(cs.name, None)
         ut.classSpec = Some(cs)
         ut
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
index db5a750a1..098881829 100644
--- a/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GoClassCompiler.scala
@@ -74,7 +74,7 @@ class GoClassCompiler(
     if (!instSpec.doc.isEmpty)
       lang.attributeDoc(instName, instSpec.doc)
     lang.instanceHeader(className, instName, dataType, instSpec.isNullable)
-    lang.instanceCheckCacheAndReturn(instName)
+    lang.instanceCheckCacheAndReturn(instName, dataType)
 
     instSpec match {
       case vi: ValueInstanceSpec =>
@@ -86,7 +86,7 @@ class GoClassCompiler(
     }
 
     lang.instanceSetCalculated(instName)
-    lang.instanceReturn(instName)
+    lang.instanceReturn(instName, dataType)
     lang.instanceFooter
   }
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
index 9629e58bc..fe275701e 100644
--- a/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RuntimeConfig.scala
@@ -26,14 +26,15 @@ case class CppRuntimeConfig(
     */
   def copyAsCpp11() = copy(
     usePragmaOnce = true,
-    pointers = CppRuntimeConfig.SharedPointers
+    pointers = CppRuntimeConfig.UniqueAndRawPointers
   )
 }
 
 object CppRuntimeConfig {
-  trait Pointers
+  sealed trait Pointers
   case object RawPointers extends Pointers
   case object SharedPointers extends Pointers
+  case object UniqueAndRawPointers extends Pointers
 }
 
 /**
diff --git a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
index f28f31c2b..a148abc43 100644
--- a/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/RustClassCompiler.scala
@@ -86,7 +86,7 @@ class RustClassCompiler(
     if (!instSpec.doc.isEmpty)
       lang.attributeDoc(instName, instSpec.doc)
     lang.instanceHeader(className, instName, dataType, instSpec.isNullable)
-    lang.instanceCheckCacheAndReturn(instName)
+    lang.instanceCheckCacheAndReturn(instName, dataType)
 
     instSpec match {
       case vi: ValueInstanceSpec =>
@@ -98,7 +98,7 @@ class RustClassCompiler(
     }
 
     lang.instanceSetCalculated(instName)
-    lang.instanceReturn(instName)
+    lang.instanceReturn(instName, dataType)
     lang.instanceFooter
   }
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index 56be8ce07..14bcf84db 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -4,7 +4,13 @@ import io.kaitai.struct.exprlang.{Ast, Expressions}
 import io.kaitai.struct.format._
 import io.kaitai.struct.translators.TypeDetector
 
-sealed trait DataType
+sealed trait DataType {
+  /**
+    * @return Data type as non-owning data type. Default implementation
+    *         always returns itself, complex types
+    */
+  def asNonOwning: DataType = this
+}
 
 /**
   * A collection of case objects and classes that are used to represent internal
@@ -89,34 +95,86 @@ object DataType {
   case object CalcBooleanType extends BooleanType
   case class ArrayType(elType: DataType) extends DataType
 
+  /**
+    * Complex data type is a data type which creation and destruction is
+    * not an atomic, built-in operation, but rather a sequence of new/delete
+    * operations. The main common trait for all complex data types is a flag
+    * that determines whether they're "owning" or "borrowed". Owning objects
+    * manage their own creation/destruction, borrowed rely on other doing
+    * that.
+    */
+  abstract class ComplexDataType extends DataType {
+    /**
+      * @return If true, this is "owning" type: for languages where data ownership
+      *         matters, this one represents primary owner of the data block, who
+      *         will be responsible for whole life cycle: creation of the object
+      *         and its destruction.
+      */
+    def isOwning: Boolean
+  }
+
+  /**
+    * Common abstract ancestor for all types which can treated as "user types".
+    * Namely, this typically means that this type has a name, may have some
+    * parameters, and forced parent expression.
+    * @param name name of the type, might include several components
+    * @param forcedParent optional parent enforcement expression
+    * @param args parameters passed into this type as extra arguments
+    */
   abstract class UserType(
     val name: List[String],
     val forcedParent: Option[Ast.expr],
     var args: Seq[Ast.expr]
-  ) extends DataType {
+  ) extends ComplexDataType {
     var classSpec: Option[ClassSpec] = None
     def isOpaque = {
       val cs = classSpec.get
       cs.isTopLevel || cs.meta.isOpaque
     }
+
+    override def asNonOwning: UserType = {
+      if (!isOwning) {
+        this
+      } else {
+        val r = CalcUserType(name, forcedParent, args)
+        r.classSpec = classSpec
+        r
+      }
+    }
   }
   case class UserTypeInstream(
     _name: List[String],
     _forcedParent: Option[Ast.expr],
     _args: Seq[Ast.expr] = Seq()
-  ) extends UserType(_name, _forcedParent, _args)
+  ) extends UserType(_name, _forcedParent, _args) {
+    def isOwning = true
+  }
   case class UserTypeFromBytes(
     _name: List[String],
     _forcedParent: Option[Ast.expr],
     _args: Seq[Ast.expr] = Seq(),
     bytes: BytesType,
     override val process: Option[ProcessExpr]
-  ) extends UserType(_name, _forcedParent, _args) with Processing
+  ) extends UserType(_name, _forcedParent, _args) with Processing {
+    def isOwning = true
+  }
+  case class CalcUserType(
+    _name: List[String],
+    _forcedParent: Option[Ast.expr],
+    _args: Seq[Ast.expr] = Seq()
+  ) extends UserType(_name, _forcedParent, _args) {
+    def isOwning = false
+  }
 
   val USER_TYPE_NO_PARENT = Ast.expr.Bool(false)
 
   case object AnyType extends DataType
-  case object KaitaiStructType extends DataType
+  case object KaitaiStructType extends ComplexDataType {
+    def isOwning = true
+  }
+  case object CalcKaitaiStructType extends ComplexDataType {
+    def isOwning = false
+  }
   case object KaitaiStreamType extends DataType
 
   case class EnumType(name: List[String], basedOn: IntType) extends DataType {
diff --git a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala
index 2a6a34dab..7e75127c9 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala
@@ -1,7 +1,8 @@
 package io.kaitai.struct.format
 
 import io.kaitai.struct.datatype.DataType
-import io.kaitai.struct.datatype.DataType.{KaitaiStructType, UserTypeInstream}
+import io.kaitai.struct.datatype.DataType._
+
 import scala.collection.mutable
 
 /**
@@ -53,8 +54,8 @@ case class ClassSpec(
   var seqSize: Sized = NotCalculatedSized
 
   def parentType: DataType = parentClass match {
-    case UnknownClassSpec | GenericStructClassSpec => KaitaiStructType
-    case t: ClassSpec => UserTypeInstream(t.name, None)
+    case UnknownClassSpec | GenericStructClassSpec => CalcKaitaiStructType
+    case t: ClassSpec => CalcUserType(t.name, None)
   }
 
   /**
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
index 73b7e9e2f..37b03722e 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
@@ -502,14 +502,14 @@ class CSharpCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("}")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"if (${flagForInstName(instName)})")
     out.inc
-    instanceReturn(instName)
+    instanceReturn(instName, dataType)
     out.dec
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)};")
   }
 
@@ -606,7 +606,7 @@ object CSharpCompiler extends LanguageCompilerStatic
       case _: BytesType => "byte[]"
 
       case AnyType => "object"
-      case KaitaiStructType => kstructName
+      case KaitaiStructType | CalcKaitaiStructType => kstructName
       case KaitaiStreamType => kstreamName
 
       case t: UserType => types2class(t.name)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 0e5bf3c31..3aa09b57a 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -1,5 +1,6 @@
 package io.kaitai.struct.languages
 
+import io.kaitai.struct.CppRuntimeConfig._
 import io.kaitai.struct._
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.datatype.{CalcEndian, DataType, FixedEndian, InheritedEndian}
@@ -23,7 +24,7 @@ class CppCompiler(
   val importListSrc = new ImportList
   val importListHdr = new ImportList
 
-  override val translator = new CppTranslator(typeProvider, importListSrc)
+  override val translator = new CppTranslator(typeProvider, importListSrc, config)
   val outSrcHeader = new StringLanguageOutputWriter(indent)
   val outHdrHeader = new StringLanguageOutputWriter(indent)
   val outSrc = new StringLanguageOutputWriter(indent)
@@ -52,6 +53,7 @@ class CppCompiler(
   override def fileHeader(topClassName: String): Unit = {
     outSrcHeader.puts(s"// $headerComment")
     outSrcHeader.puts
+    outSrcHeader.puts("#include <memory>")
     outSrcHeader.puts("#include \"" + outFileName(topClassName) + ".h\"")
     outSrcHeader.puts
 
@@ -69,6 +71,13 @@ class CppCompiler(
 
     importListHdr.add("stdint.h")
 
+    config.cppConfig.pointers match {
+      case SharedPointers | UniqueAndRawPointers =>
+        importListHdr.add("memory")
+      case RawPointers =>
+        // no extra includes
+    }
+
     // API compatibility check
     val minVer = KSVersion.minimalRuntime.toInt
     outHdr.puts
@@ -107,8 +116,15 @@ class CppCompiler(
   }
 
   override def classHeader(name: List[String]): Unit = {
+    val className = types2class(List(name.last))
+
+    val extraInherits = config.cppConfig.pointers match {
+      case RawPointers | UniqueAndRawPointers => ""
+      case SharedPointers => s", std::enable_shared_from_this<$className>"
+    }
+
     outHdr.puts
-    outHdr.puts(s"class ${types2class(List(name.last))} : public $kstructName {")
+    outHdr.puts(s"class $className : public $kstructName$extraInherits {")
     outHdr.inc
     accessMode = PrivateAccess
     ensureMode(PublicAccess)
@@ -147,33 +163,46 @@ class CppCompiler(
       s"${kaitaiType2NativeType(p.dataType)} ${paramName(p.id)}"
     ), "", ", ", ", ")
 
+    val classNameBrief = types2class(List(name.last))
+
     // Parameter names
     val pIo = paramName(IoIdentifier)
     val pParent = paramName(ParentIdentifier)
     val pRoot = paramName(RootIdentifier)
 
     // Types
-    val tIo = s"$kstreamName*"
+    val tIo = kaitaiType2NativeType(KaitaiStreamType)
     val tParent = kaitaiType2NativeType(parentType)
-    val tRoot = s"${types2class(rootClassName)}*"
+    val tRoot = kaitaiType2NativeType(CalcUserType(rootClassName, None))
 
     outHdr.puts
-    outHdr.puts(s"${types2class(List(name.last))}($paramsArg" +
+    outHdr.puts(s"$classNameBrief($paramsArg" +
       s"$tIo $pIo, " +
-      s"$tParent $pParent = 0, " +
-      s"$tRoot $pRoot = 0$endianSuffixHdr);"
+      s"$tParent $pParent = $nullPtr, " +
+      s"$tRoot $pRoot = $nullPtr$endianSuffixHdr);"
     )
 
     outSrc.puts
-    outSrc.puts(s"${types2class(name)}::${types2class(List(name.last))}($paramsArg" +
+    outSrc.puts(s"${types2class(name)}::$classNameBrief($paramsArg" +
       s"$tIo $pIo, " +
       s"$tParent $pParent, " +
       s"$tRoot $pRoot$endianSuffixSrc) : $kstructName($pIo) {"
     )
     outSrc.inc
+
+    // In shared pointers mode, this is required to be able to work with shared pointers to this
+    // in a constructor. This is obviously a hack and not a good practice.
+    // https://forum.libcinder.org/topic/solution-calling-shared-from-this-in-the-constructor
+    if (config.cppConfig.pointers == CppRuntimeConfig.SharedPointers) {
+      outSrc.puts(s"const auto weakPtrTrick = std::shared_ptr<$classNameBrief>(this, []($classNameBrief*){});")
+    }
+
     handleAssignmentSimple(ParentIdentifier, pParent)
     handleAssignmentSimple(RootIdentifier, if (name == rootClassName) {
-      "this"
+      config.cppConfig.pointers match {
+        case RawPointers | UniqueAndRawPointers => "this"
+        case SharedPointers => "shared_from_this()"
+      }
     } else {
       pRoot
     })
@@ -271,7 +300,7 @@ class CppCompiler(
 
   override def attributeReader(attrName: Identifier, attrType: DataType, isNullable: Boolean): Unit = {
     ensureMode(PublicAccess)
-    outHdr.puts(s"${kaitaiType2NativeType(attrType)} ${publicMemberName(attrName)}() const { return ${privateMemberName(attrName)}; }")
+    outHdr.puts(s"${kaitaiType2NativeType(attrType.asNonOwning)} ${publicMemberName(attrName)}() const { return ${nonOwningPointer(attrName, attrType)}; }")
   }
 
   override def universalDoc(doc: DocSpec): Unit = {
@@ -299,7 +328,7 @@ class CppCompiler(
     attr.dataTypeComposite match {
       case _: UserType | _: ArrayType | KaitaiStreamType =>
         // data type will be pointer to user type, std::vector or stream, so we need to init it
-        outSrc.puts(s"${privateMemberName(attr.id)} = 0;")
+        outSrc.puts(s"${privateMemberName(attr.id)} = $nullPtr;")
       case _ =>
         // no init required for value types
     }
@@ -354,22 +383,24 @@ class CppCompiler(
         outSrc.puts(s"delete $ioVar;")
       }
 
-      // main member contents
-      if (needsDestruction(innerType)) {
-        val arrVar = privateMemberName(id)
+      if (config.cppConfig.pointers == CppRuntimeConfig.RawPointers) {
+        // main member contents
+        if (needsDestruction(innerType)) {
+          val arrVar = privateMemberName(id)
+
+          // C++ specific substitution: AnyType results from generic struct + raw bytes
+          // so we would assume that only generic struct needs to be cleaned up
+          val realType = innerType match {
+            case AnyType => KaitaiStructType
+            case _ => innerType
+          }
 
-        // C++ specific substitution: AnyType results from generic struct + raw bytes
-        // so we would assume that only generic struct needs to be cleaned up
-        val realType = innerType match {
-          case AnyType => KaitaiStructType
-          case _ => innerType
+          destructVector(kaitaiType2NativeType(realType), arrVar)
         }
 
-        destructVector(kaitaiType2NativeType(realType), arrVar)
+        // main member is a std::vector of something, always needs destruction
+        outSrc.puts(s"delete ${privateMemberName(id)};")
       }
-
-      // main member is a std::vector of something, always needs destruction
-      outSrc.puts(s"delete ${privateMemberName(id)};")
     } else {
       // raw is just a string, no need to cleanup => we ignore `hasRaw`
 
@@ -377,7 +408,7 @@ class CppCompiler(
       if (hasIO)
         outSrc.puts(s"delete ${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))};")
 
-      if (needsDestruction(innerType))
+      if (config.cppConfig.pointers == CppRuntimeConfig.RawPointers && needsDestruction(innerType))
         outSrc.puts(s"delete ${privateMemberName(id)};")
     }
   }
@@ -628,9 +659,13 @@ class CppCompiler(
           ""
         } else {
           val parent = t.forcedParent match {
-            case Some(USER_TYPE_NO_PARENT) => "0"
+            case Some(USER_TYPE_NO_PARENT) => nullPtr
             case Some(fp) => translator.translate(fp)
-            case None => "this"
+            case None =>
+              config.cppConfig.pointers match {
+                case RawPointers | UniqueAndRawPointers => "this"
+                case SharedPointers => s"shared_from_this()"
+              }
           }
           val addEndian = t.classSpec.get.meta.endian match {
             case Some(InheritedEndian) => ", m__is_le"
@@ -638,7 +673,15 @@ class CppCompiler(
           }
           s", $parent, ${privateMemberName(RootIdentifier)}$addEndian"
         }
-        s"new ${types2class(t.name)}($addParams$io$addArgs)"
+        config.cppConfig.pointers match {
+          case RawPointers =>
+            s"new ${types2class(t.name)}($addParams$io$addArgs)"
+          case SharedPointers =>
+            s"std::make_shared<${types2class(t.name)}>($addParams$io$addArgs)"
+          case UniqueAndRawPointers =>
+            importListSrc.add("memory")
+            s"std::make_unique<${types2class(t.name)}>($addParams$io$addArgs)"
+        }
     }
   }
 
@@ -741,10 +784,10 @@ class CppCompiler(
 
   override def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit = {
     ensureMode(PublicAccess)
-    outHdr.puts(s"${kaitaiType2NativeType(dataType)} ${publicMemberName(instName)}();")
+    outHdr.puts(s"${kaitaiType2NativeType(dataType.asNonOwning)} ${publicMemberName(instName)}();")
 
     outSrc.puts
-    outSrc.puts(s"${kaitaiType2NativeType(dataType, true)} ${types2class(className)}::${publicMemberName(instName)}() {")
+    outSrc.puts(s"${kaitaiType2NativeType(dataType.asNonOwning, true)} ${types2class(className)}::${publicMemberName(instName)}() {")
     outSrc.inc
   }
 
@@ -753,16 +796,15 @@ class CppCompiler(
     outSrc.puts("}")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     outSrc.puts(s"if (${calculatedFlagForName(instName)})")
     outSrc.inc
-    instanceReturn(instName)
+    instanceReturn(instName, dataType)
     outSrc.dec
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
-    outSrc.puts(s"return ${privateMemberName(instName)};")
-  }
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit =
+    outSrc.puts(s"return ${nonOwningPointer(instName, attrType)};")
 
   override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = {
     val enumClass = types2class(List(enumName))
@@ -838,6 +880,32 @@ class CppCompiler(
   }
 
   override def type2class(className: String): String = CppCompiler.type2class(className)
+
+  def kaitaiType2NativeType(attrType: DataType, absolute: Boolean = false): String =
+    CppCompiler.kaitaiType2NativeType(config.cppConfig, attrType, absolute)
+
+  def nullPtr: String = config.cppConfig.pointers match {
+    case RawPointers => "0"
+    case SharedPointers | UniqueAndRawPointers => "nullptr"
+  }
+
+  def nonOwningPointer(attrName: Identifier, attrType: DataType): String = {
+    config.cppConfig.pointers match {
+      case RawPointers =>
+        privateMemberName(attrName)
+      case UniqueAndRawPointers =>
+        attrType match {
+          case ut: UserType =>
+            if (ut.isOwning) {
+              s"${privateMemberName(attrName)}.get()"
+            } else {
+              privateMemberName(attrName)
+            }
+          case _ =>
+            privateMemberName(attrName)
+        }
+    }
+  }
 }
 
 object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
@@ -849,7 +917,7 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
   override def kstructName = "kaitai::kstruct"
   override def kstreamName = "kaitai::kstream"
 
-  def kaitaiType2NativeType(attrType: DataType, absolute: Boolean = false): String = {
+  def kaitaiType2NativeType(config: CppRuntimeConfig, attrType: DataType, absolute: Boolean = false): String = {
     attrType match {
       case Int1Type(false) => "uint8_t"
       case IntMultiType(false, Width2, _) => "uint16_t"
@@ -879,7 +947,12 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
         } else {
           t.name
         })
-        s"$typeStr*"
+        config.pointers match {
+          case RawPointers => s"$typeStr*"
+          case SharedPointers => s"std::shared_ptr<$typeStr>"
+          case UniqueAndRawPointers =>
+            if (t.isOwning) s"std::unique_ptr<$typeStr>" else s"$typeStr*"
+        }
 
       case t: EnumType =>
         types2class(if (absolute) {
@@ -888,13 +961,22 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
           t.name
         })
 
-      case ArrayType(inType) => s"std::vector<${kaitaiType2NativeType(inType, absolute)}>*"
+      case ArrayType(inType) => s"std::vector<${kaitaiType2NativeType(config, inType, absolute)}>*"
 
       case KaitaiStreamType => s"$kstreamName*"
-      case KaitaiStructType => s"$kstructName*"
+      case KaitaiStructType => config.pointers match {
+        case RawPointers => s"$kstructName*"
+        case SharedPointers => s"std::shared_ptr<$kstructName>"
+        case UniqueAndRawPointers => s"std::unique_ptr<$kstructName>"
+      }
+      case CalcKaitaiStructType => config.pointers match {
+        case RawPointers => s"$kstructName*"
+        case SharedPointers => s"std::shared_ptr<$kstructName>"
+        case UniqueAndRawPointers => s"$kstructName*"
+      }
 
       case SwitchType(on, cases) =>
-        kaitaiType2NativeType(TypeDetector.combineTypes(
+        kaitaiType2NativeType(config, TypeDetector.combineTypes(
           // C++ does not have a concept of AnyType, and common use case "lots of incompatible UserTypes
           // for cases + 1 BytesType for else" combined would result in exactly AnyType - so we try extra
           // hard to avoid that here with this pre-filtering. In C++, "else" case with raw byte array would
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index c3a4e7c50..9109a436b 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -358,14 +358,14 @@ class GoCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts(s"${privateMemberName(instName)} = $converted")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"if (this.${calculatedFlagForName(instName)}) {")
     out.inc
-    instanceReturn(instName)
+    instanceReturn(instName, dataType)
     universalFooter
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)}, nil")
   }
 
@@ -460,7 +460,7 @@ object GoCompiler extends LanguageCompilerStatic
 
       case AnyType => "interface{}"
       case KaitaiStreamType => "*" + kstreamName
-      case KaitaiStructType => kstructName
+      case KaitaiStructType | CalcKaitaiStructType => kstructName
 
       case t: UserType => "*" + types2class(t.classSpec match {
         case Some(cs) => cs.name
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index e74c581f7..dc161f095 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -579,14 +579,14 @@ class JavaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"if (${privateMemberName(instName)} != null)")
     out.inc
-    instanceReturn(instName)
+    instanceReturn(instName, dataType)
     out.dec
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)};")
   }
 
@@ -719,7 +719,7 @@ object JavaCompiler extends LanguageCompilerStatic
 
       case AnyType => "Object"
       case KaitaiStreamType => kstreamName
-      case KaitaiStructType => kstructName
+      case KaitaiStructType | CalcKaitaiStructType => kstructName
 
       case t: UserType => types2class(t.name)
       case EnumType(name, _) => types2class(name)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
index a3842f748..bcdef1942 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaScriptCompiler.scala
@@ -503,14 +503,14 @@ class JavaScriptCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("});")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"if (${privateMemberName(instName)} !== undefined)")
     out.inc
-    instanceReturn(instName)
+    instanceReturn(instName, dataType)
     out.dec
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)};")
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala
index ebba1a88c..3330b3c71 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala
@@ -263,15 +263,15 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("end")
     out.puts
   }
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"if self.${idToStr(instName)} ~= nil then")
     out.inc
-    instanceReturn(instName)
+    instanceReturn(instName, dataType)
     out.dec
     out.puts("end")
     out.puts
   }
-  override def instanceReturn(instName: InstanceIdentifier): Unit =
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit =
     out.puts(s"return ${privateMemberName(instName)}")
 
   override def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit = {
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
index 9b4847b90..111c718e0 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PHPCompiler.scala
@@ -387,14 +387,14 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"if (${privateMemberName(instName)} !== null)")
     out.inc
-    instanceReturn(instName)
+    instanceReturn(instName, dataType)
     out.dec
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)};")
   }
 
@@ -458,7 +458,7 @@ class PHPCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
       case ArrayType(_) => "array"
 
-      case KaitaiStructType => kstructName
+      case KaitaiStructType | CalcKaitaiStructType => kstructName
       case KaitaiStreamType => kstreamName
     }
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
index fdf020569..c6a24915b 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PerlCompiler.scala
@@ -363,11 +363,11 @@ class PerlCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts("my ($self) = @_;")
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)} if (${privateMemberName(instName)});")
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)};")
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index 93f351b87..bac7a846b 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -426,15 +426,15 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"if hasattr(self, '${idToStr(instName)}'):")
     out.inc
-    instanceReturn(instName)
+    instanceReturn(instName, dataType)
     out.dec
     out.puts
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     // not very efficient, probably should be some other way to do that, but for now it will do:
     // workaround to avoid Python generating an "AttributeError: instance has no attribute"
     out.puts(s"return ${privateMemberName(instName)} if hasattr(self, '${idToStr(instName)}') else None")
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
index 3788276bb..b832ab3ae 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RubyCompiler.scala
@@ -404,11 +404,11 @@ class RubyCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)} unless ${privateMemberName(instName)}.nil?")
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     out.puts(privateMemberName(instName))
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index fc9965300..d5e057370 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -440,7 +440,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.inc
   }
 
-  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit = {
     out.puts(s"if let Some(x) = ${privateMemberName(instName)} {")
     out.inc
     out.puts("return x;")
@@ -449,7 +449,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.puts
   }
 
-  override def instanceReturn(instName: InstanceIdentifier): Unit = {
+  override def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit = {
     out.puts(s"return ${privateMemberName(instName)};")
   }
 
@@ -532,7 +532,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case ArrayType(inType) => s"Vec<${kaitaiType2NativeType(inType)}>"
 
       case KaitaiStreamType => s"Option<Box<KaitaiStream>>"
-      case KaitaiStructType => s"Option<Box<KaitaiStruct>>"
+      case KaitaiStructType | CalcKaitaiStructType => s"Option<Box<KaitaiStruct>>"
       
       case SwitchType(on, cases) => kaitaiType2NativeType(TypeDetector.combineTypes(cases.values))
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
index 922e94294..f41214463 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompiler.scala
@@ -125,8 +125,8 @@ abstract class LanguageCompiler(
   def instanceDeclaration(attrName: InstanceIdentifier, attrType: DataType, isNullable: Boolean): Unit = attributeDeclaration(attrName, attrType, isNullable)
   def instanceHeader(className: List[String], instName: InstanceIdentifier, dataType: DataType, isNullable: Boolean): Unit
   def instanceFooter: Unit
-  def instanceCheckCacheAndReturn(instName: InstanceIdentifier): Unit
-  def instanceReturn(instName: InstanceIdentifier): Unit
+  def instanceCheckCacheAndReturn(instName: InstanceIdentifier, dataType: DataType): Unit
+  def instanceReturn(instName: InstanceIdentifier, attrType: DataType): Unit
   def instanceCalculate(instName: Identifier, dataType: DataType, value: Ast.expr)
 
   def enumDeclaration(curClass: List[String], enumName: String, enumColl: Seq[(Long, EnumValueSpec)]): Unit
diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala
index 50cf82c30..eb1937025 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ValueTypesDeriver.scala
@@ -24,7 +24,7 @@ class ValueTypesDeriver(specs: ClassSpecs, topClass: ClassSpec) {
             vi.dataType match {
               case None =>
                 try {
-                  val viType = detector.detectType(vi.value)
+                  val viType = detector.detectType(vi.value).asNonOwning
                   vi.dataType = Some(viType)
                   Log.typeProcValue.info(() => s"${instName.name} derived type: $viType")
                   hasChanged = true
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index f7eb61606..59e1b66f4 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -2,15 +2,16 @@ package io.kaitai.struct.translators
 
 import java.nio.charset.Charset
 
+import io.kaitai.struct.CppRuntimeConfig.{RawPointers, SharedPointers}
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.format.Identifier
 import io.kaitai.struct.languages.CppCompiler
-import io.kaitai.struct.{ImportList, Utils}
+import io.kaitai.struct.{ImportList, RuntimeConfig, Utils}
 
-class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends BaseTranslator(provider) {
+class CppTranslator(provider: TypeProvider, importListSrc: ImportList, config: RuntimeConfig) extends BaseTranslator(provider) {
   val CHARSET_UTF8 = Charset.forName("UTF-8")
 
   /**
@@ -147,7 +148,19 @@ class CppTranslator(provider: TypeProvider, importListSrc: ImportList) extends B
   override def doIfExp(condition: expr, ifTrue: expr, ifFalse: expr): String =
     s"((${translate(condition)}) ? (${translate(ifTrue)}) : (${translate(ifFalse)}))"
   override def doCast(value: Ast.expr, typeName: DataType): String =
-    s"static_cast<${CppCompiler.kaitaiType2NativeType(typeName)}>(${translate(value)})"
+    config.cppConfig.pointers match {
+      case RawPointers =>
+        cppStaticCast(value, typeName)
+      case SharedPointers =>
+        typeName match {
+          case ut: UserType =>
+            s"std::static_pointer_cast<${CppCompiler.types2class(ut.classSpec.get.name)}>(${translate(value)})"
+          case _ => cppStaticCast(value, typeName)
+        }
+    }
+
+  def cppStaticCast(value: Ast.expr, typeName: DataType): String =
+    s"static_cast<${CppCompiler.kaitaiType2NativeType(config.cppConfig, typeName)}>(${translate(value)})"
 
   // Predefined methods of various types
   override def strToInt(s: expr, base: expr): String = {

From 237330ab6100f630fbdf58959a415e9e60d22957 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sat, 22 Dec 2018 20:08:01 +0000
Subject: [PATCH 195/230] CppCompiler: refactored std::move wrap out of
 push_back calls

---
 .../scala/io/kaitai/struct/languages/CppCompiler.scala | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 3aa09b57a..3870e8abb 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -558,7 +558,7 @@ class CppCompiler(
   }
 
   override def handleAssignmentRepeatEos(id: Identifier, expr: String): Unit = {
-    outSrc.puts(s"${privateMemberName(id)}->push_back($expr);")
+    outSrc.puts(s"${privateMemberName(id)}->push_back(${stdMoveWrap(expr)});")
   }
 
   override def condRepeatEosFooter: Unit = {
@@ -589,7 +589,7 @@ class CppCompiler(
   }
 
   override def handleAssignmentRepeatExpr(id: Identifier, expr: String): Unit = {
-    outSrc.puts(s"${privateMemberName(id)}->push_back($expr);")
+    outSrc.puts(s"${privateMemberName(id)}->push_back(${stdMoveWrap(expr)});")
   }
 
   override def condRepeatExprFooter: Unit = {
@@ -620,6 +620,7 @@ class CppCompiler(
       ("", translator.doName(Identifier.ITERATOR))
     }
     outSrc.puts(s"$typeDecl$tempVar = $expr;")
+
     outSrc.puts(s"${privateMemberName(id)}->push_back($tempVar);")
   }
 
@@ -906,6 +907,11 @@ class CppCompiler(
         }
     }
   }
+
+  def stdMoveWrap(expr: String): String = config.cppConfig.pointers match {
+    case UniqueAndRawPointers => s"std::move($expr)"
+    case _ => expr
+  }
 }
 
 object CppCompiler extends LanguageCompilerStatic with StreamStructNames {

From edff49788ec80d292574740f27a2d6eba0d35e58 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 23 Dec 2018 02:08:09 +0000
Subject: [PATCH 196/230] Extracted RepeatSpec parsing into separate file,
 optimized public API

---
 .../io/kaitai/struct/format/AttrSpec.scala    | 49 +-----------------
 .../io/kaitai/struct/format/RepeatSpec.scala  | 51 +++++++++++++++++++
 2 files changed, 52 insertions(+), 48 deletions(-)
 create mode 100644 shared/src/main/scala/io/kaitai/struct/format/RepeatSpec.scala

diff --git a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
index 613898f06..00a7d54f0 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
@@ -10,12 +10,6 @@ import io.kaitai.struct.exprlang.{Ast, Expressions}
 
 import scala.collection.JavaConversions._
 
-sealed trait RepeatSpec
-case class RepeatExpr(expr: Ast.expr) extends RepeatSpec
-case class RepeatUntil(expr: Ast.expr) extends RepeatSpec
-case object RepeatEos extends RepeatSpec
-case object NoRepeat extends RepeatSpec
-
 case class ConditionalSpec(ifExpr: Option[Ast.expr], repeat: RepeatSpec)
 
 trait AttrLikeSpec extends MemberSpec {
@@ -177,9 +171,6 @@ object AttrSpec {
     val sizeEos = ParseUtils.getOptValueBool(srcMap, "size-eos", path).getOrElse(false)
     val ifExpr = ParseUtils.getOptValueExpression(srcMap, "if", path)
     val encoding = ParseUtils.getOptValueStr(srcMap, "encoding", path)
-    val repeat = ParseUtils.getOptValueStr(srcMap, "repeat", path)
-    val repeatExpr = ParseUtils.getOptValueExpression(srcMap, "repeat-expr", path)
-    val repeatUntil = ParseUtils.getOptValueExpression(srcMap, "repeat-until", path)
     val terminator = ParseUtils.getOptValueInt(srcMap, "terminator", path)
     val consume = ParseUtils.getOptValueBool(srcMap, "consume", path).getOrElse(true)
     val include = ParseUtils.getOptValueBool(srcMap, "include", path).getOrElse(false)
@@ -216,7 +207,7 @@ object AttrSpec {
         }
     }
 
-    val (repeatSpec, legalRepeatKeys) = parseRepeat(repeat, repeatExpr, repeatUntil, path)
+    val (repeatSpec, legalRepeatKeys) = RepeatSpec.fromYaml(srcMap, path)
 
     val legalKeys = LEGAL_KEYS ++ legalRepeatKeys ++ (dataType match {
       case _: BytesType => LEGAL_KEYS_BYTES
@@ -304,42 +295,4 @@ object AttrSpec {
 
     SwitchType(on, cases ++ addCases)
   }
-
-  private def parseRepeat(
-    repeat: Option[String],
-    rExpr: Option[Ast.expr],
-    rUntil: Option[Ast.expr],
-    path: List[String]
-  ): (RepeatSpec, Set[String]) = {
-    repeat match {
-      case None =>
-        (NoRepeat, Set())
-      case Some("until") =>
-        val spec = rUntil match {
-          case Some(expr) => RepeatUntil(expr)
-          case None =>
-            throw new YAMLParseException(
-              "`repeat: until` requires a `repeat-until` expression",
-              path ++ List("repeat")
-            )
-        }
-        (spec, Set("repeat-until"))
-      case Some("expr") =>
-        val spec = rExpr match {
-          case Some(expr) => RepeatExpr(expr)
-          case None =>
-            throw new YAMLParseException(
-              "`repeat: expr` requires a `repeat-expr` expression",
-              path ++ List("repeat")
-            )
-        }
-        (spec, Set("repeat-expr"))
-      case Some("eos") =>
-        (RepeatEos, Set())
-      case Some(other) =>
-        throw YAMLParseException.badDictValue(
-          Set("until", "expr", "eos"), other, path ++ List("repeat")
-        )
-    }
-  }
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/format/RepeatSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/RepeatSpec.scala
new file mode 100644
index 000000000..01187c090
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/format/RepeatSpec.scala
@@ -0,0 +1,51 @@
+package io.kaitai.struct.format
+
+import io.kaitai.struct.exprlang.Ast
+
+sealed trait RepeatSpec
+case class RepeatExpr(expr: Ast.expr) extends RepeatSpec
+case class RepeatUntil(expr: Ast.expr) extends RepeatSpec
+case object RepeatEos extends RepeatSpec
+case object NoRepeat extends RepeatSpec
+
+object RepeatSpec {
+  def fromYaml(
+    srcMap: Map[String, Any],
+    path: List[String]
+  ): (RepeatSpec, Set[String]) = {
+    val repeat = ParseUtils.getOptValueStr(srcMap, "repeat", path)
+    val repeatExpr = ParseUtils.getOptValueExpression(srcMap, "repeat-expr", path)
+    val repeatUntil = ParseUtils.getOptValueExpression(srcMap, "repeat-until", path)
+
+    repeat match {
+      case None =>
+        (NoRepeat, Set())
+      case Some("until") =>
+        val spec = repeatUntil match {
+          case Some(expr) => RepeatUntil(expr)
+          case None =>
+            throw new YAMLParseException(
+              "`repeat: until` requires a `repeat-until` expression",
+              path ++ List("repeat")
+            )
+        }
+        (spec, Set("repeat-until"))
+      case Some("expr") =>
+        val spec = repeatExpr match {
+          case Some(expr) => RepeatExpr(expr)
+          case None =>
+            throw new YAMLParseException(
+              "`repeat: expr` requires a `repeat-expr` expression",
+              path ++ List("repeat")
+            )
+        }
+        (spec, Set("repeat-expr"))
+      case Some("eos") =>
+        (RepeatEos, Set())
+      case Some(other) =>
+        throw YAMLParseException.badDictValue(
+          Set("until", "expr", "eos"), other, path ++ List("repeat")
+        )
+    }
+  }
+}
\ No newline at end of file

From 6ef4f11c0888724b09e60cb3cfc33ba58892c4fb Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 24 Dec 2018 12:42:49 +0000
Subject: [PATCH 197/230] CppCompiler: better SwitchType/CalcKaitaiStruct
 support

---
 .../io/kaitai/struct/TypeProcessor.scala      |  4 +--
 .../io/kaitai/struct/datatype/DataType.scala  |  5 +++-
 .../io/kaitai/struct/format/AttrSpec.scala    |  2 +-
 .../io/kaitai/struct/format/ClassSpec.scala   | 15 +++++++++++
 .../struct/languages/CSharpCompiler.scala     |  2 +-
 .../kaitai/struct/languages/CppCompiler.scala | 14 ++++++++---
 .../kaitai/struct/languages/GoCompiler.scala  |  2 +-
 .../struct/languages/JavaCompiler.scala       |  2 +-
 .../struct/languages/RustCompiler.scala       |  4 +--
 .../struct/precompile/ResolveTypes.scala      |  4 +--
 .../struct/translators/TypeDetector.scala     | 25 ++++++++++++++-----
 11 files changed, 58 insertions(+), 21 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
index 7aaafaae4..92314ae9d 100644
--- a/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
+++ b/shared/src/main/scala/io/kaitai/struct/TypeProcessor.scala
@@ -36,8 +36,8 @@ object TypeProcessor {
         } else {
           List()
         }
-      case SwitchType(_, cases) =>
-        cases.flatMap { case (_, ut) =>
+      case st: SwitchType =>
+        st.cases.flatMap { case (_, ut) =>
           getOpaqueDataTypes(ut)
         }
       case _ =>
diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index 14bcf84db..c814b4b89 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -171,6 +171,7 @@ object DataType {
   case object AnyType extends DataType
   case object KaitaiStructType extends ComplexDataType {
     def isOwning = true
+    override def asNonOwning: DataType = CalcKaitaiStructType
   }
   case object CalcKaitaiStructType extends ComplexDataType {
     def isOwning = false
@@ -181,7 +182,7 @@ object DataType {
     var enumSpec: Option[EnumSpec] = None
   }
 
-  case class SwitchType(on: Ast.expr, cases: Map[Ast.expr, DataType]) extends DataType {
+  case class SwitchType(on: Ast.expr, cases: Map[Ast.expr, DataType], isOwning: Boolean = true) extends ComplexDataType {
     def combinedType: DataType = TypeDetector.combineTypes(cases.values)
 
     /**
@@ -218,6 +219,8 @@ object DataType {
       cases.values.exists((t) =>
         t.isInstanceOf[UserTypeFromBytes] || t.isInstanceOf[BytesType]
       )
+
+    override def asNonOwning: DataType = SwitchType(on, cases, false)
   }
 
   object SwitchType {
diff --git a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
index 00a7d54f0..60d547e89 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/AttrSpec.scala
@@ -214,7 +214,7 @@ object AttrSpec {
       case _: StrFromBytesType => LEGAL_KEYS_STR
       case _: UserType => LEGAL_KEYS_BYTES
       case EnumType(_, _) => LEGAL_KEYS_ENUM
-      case SwitchType(on, cases) => LEGAL_KEYS_BYTES
+      case _: SwitchType => LEGAL_KEYS_BYTES
       case _ => Set()
     })
 
diff --git a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala
index 7e75127c9..60582238d 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/ClassSpec.scala
@@ -68,6 +68,21 @@ case class ClassSpec(
       typeSpec.forEachRec(proc)
     }
   }
+
+  override def equals(obj: Any): Boolean = obj match {
+    case other: ClassSpec =>
+      path == other.path &&
+      isTopLevel == other.isTopLevel &&
+      meta == other.meta &&
+      doc == other.doc &&
+      params == other.params &&
+      seq == other.seq &&
+      types == other.types &&
+      instances == other.instances &&
+      enums == other.enums &&
+      name == other.name
+    case _ => false
+  }
 }
 
 object ClassSpec {
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
index 37b03722e..c4985ed24 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CSharpCompiler.scala
@@ -614,7 +614,7 @@ object CSharpCompiler extends LanguageCompilerStatic
 
       case ArrayType(inType) => s"List<${kaitaiType2NativeType(inType)}>"
 
-      case SwitchType(_, cases) => kaitaiType2NativeType(TypeDetector.combineTypes(cases.values))
+      case st: SwitchType => kaitaiType2NativeType(st.combinedType)
     }
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 3870e8abb..88ed361cc 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -981,14 +981,20 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
         case UniqueAndRawPointers => s"$kstructName*"
       }
 
-      case SwitchType(on, cases) =>
-        kaitaiType2NativeType(config, TypeDetector.combineTypes(
+      case st: SwitchType =>
+        val ct1 = TypeDetector.combineTypes(
           // C++ does not have a concept of AnyType, and common use case "lots of incompatible UserTypes
           // for cases + 1 BytesType for else" combined would result in exactly AnyType - so we try extra
           // hard to avoid that here with this pre-filtering. In C++, "else" case with raw byte array would
           // be available through _raw_* attribute anyway.
-          cases.filterNot { case (caseExpr, caseValue) => caseExpr == SwitchType.ELSE_CONST }.values
-        ), absolute)
+          st.cases.filterNot { case (caseExpr, caseValue) => caseExpr == SwitchType.ELSE_CONST }.values
+        )
+        val ct2 = if (st.isOwning) {
+          ct1
+        } else {
+          ct1.asNonOwning
+        }
+        kaitaiType2NativeType(config, ct2, absolute)
     }
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
index 9109a436b..7d5d0e5eb 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/GoCompiler.scala
@@ -470,7 +470,7 @@ object GoCompiler extends LanguageCompilerStatic
 
       case ArrayType(inType) => s"[]${kaitaiType2NativeType(inType)}"
 
-      case SwitchType(_, cases) => kaitaiType2NativeType(TypeDetector.combineTypes(cases.values))
+      case st: SwitchType => kaitaiType2NativeType(st.combinedType)
     }
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index dc161f095..1de08c004 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -770,7 +770,7 @@ object JavaCompiler extends LanguageCompilerStatic
 
       case ArrayType(inType) => s"ArrayList<${kaitaiType2JavaTypeBoxed(inType)}>"
 
-      case SwitchType(_, cases) => kaitaiType2JavaTypeBoxed(TypeDetector.combineTypes(cases.values))
+      case st: SwitchType => kaitaiType2JavaTypeBoxed(st.combinedType)
     }
   }
 
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
index d5e057370..38886e3b4 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/RustCompiler.scala
@@ -534,7 +534,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case KaitaiStreamType => s"Option<Box<KaitaiStream>>"
       case KaitaiStructType | CalcKaitaiStructType => s"Option<Box<KaitaiStruct>>"
       
-      case SwitchType(on, cases) => kaitaiType2NativeType(TypeDetector.combineTypes(cases.values))
+      case st: SwitchType => kaitaiType2NativeType(st.combinedType)
     }
   }
   
@@ -570,7 +570,7 @@ class RustCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
       case KaitaiStreamType => "None"
       case KaitaiStructType => "None"
       
-      case SwitchType(on, cases) => ""
+      case _: SwitchType => ""
       // TODO
     }
   }
diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
index 4ee06ecc0..d221c1890 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
@@ -45,8 +45,8 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) {
           val err = new EnumNotFoundError(et.name.mkString("::"), curClass)
           throw new YAMLParseException(err.getMessage, path)
         }
-      case SwitchType(_, cases) =>
-        cases.foreach { case (caseName, ut) =>
+      case st: SwitchType =>
+        st.cases.foreach { case (caseName, ut) =>
           resolveUserType(curClass, ut, path ++ List("type", "cases", caseName.toString))
         }
       case _ =>
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
index e1acb3fc0..5f9aed1b5 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/TypeDetector.scala
@@ -131,7 +131,7 @@ class TypeDetector(provider: TypeProvider) {
   def detectAttributeType(value: Ast.expr, attr: Ast.identifier) = {
     val valType = detectType(value)
     valType match {
-      case KaitaiStructType =>
+      case KaitaiStructType | CalcKaitaiStructType =>
         throw new TypeMismatchError(s"called attribute '${attr.name}' on generic struct expression '$value'")
       case t: UserType =>
         t.classSpec match {
@@ -338,19 +338,31 @@ object TypeDetector {
               if (t1.name == t2.name) {
                 t1
               } else {
-                KaitaiStructType
+                if (t1.isOwning || t2.isOwning) {
+                  KaitaiStructType
+                } else {
+                  CalcKaitaiStructType
+                }
               }
             case (Some(cs1), Some(cs2)) =>
               if (cs1 == cs2) {
                 t1
               } else {
-                KaitaiStructType
+                if (t1.isOwning || t2.isOwning) {
+                  KaitaiStructType
+                } else {
+                  CalcKaitaiStructType
+                }
               }
             case (_, _) =>
-              KaitaiStructType
+              if (t1.isOwning || t2.isOwning) {
+                KaitaiStructType
+              } else {
+                CalcKaitaiStructType
+              }
           }
-        case (_: UserType, KaitaiStructType) => KaitaiStructType
-        case (KaitaiStructType, _: UserType) => KaitaiStructType
+        case (_: UserType, _: ComplexDataType) => CalcKaitaiStructType
+        case (_: ComplexDataType, _: UserType) => CalcKaitaiStructType
         case _ => AnyType
       }
     }
@@ -401,6 +413,7 @@ object TypeDetector {
         case (_: BooleanType, _: BooleanType) => true
         case (_: StrType, _: StrType) => true
         case (_: UserType, KaitaiStructType) => true
+        case (_: UserType, CalcKaitaiStructType) => true
         case (t1: UserType, t2: UserType) =>
           (t1.classSpec, t2.classSpec) match {
             case (None, None) =>

From 00ca5f763ca6e34ca38f71268ab028ab81d62a4c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 24 Dec 2018 22:55:05 +0000
Subject: [PATCH 198/230] CppCompiler: fix ->get() generation for owning
 KaitaiStruct members

---
 .../main/scala/io/kaitai/struct/languages/CppCompiler.scala   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 88ed361cc..5b003f0a4 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -896,8 +896,8 @@ class CppCompiler(
         privateMemberName(attrName)
       case UniqueAndRawPointers =>
         attrType match {
-          case ut: UserType =>
-            if (ut.isOwning) {
+          case t: ComplexDataType =>
+            if (t.isOwning) {
               s"${privateMemberName(attrName)}.get()"
             } else {
               privateMemberName(attrName)

From 974842f8680604fb53776e6b5fb2a37ab8058da7 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 25 Dec 2018 01:54:32 +0000
Subject: [PATCH 199/230] CppCompiler: use C++11 construct instead of C++14
 std::make_unique

---
 .../main/scala/io/kaitai/struct/languages/CppCompiler.scala   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 5b003f0a4..26e4e8794 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -681,7 +681,9 @@ class CppCompiler(
             s"std::make_shared<${types2class(t.name)}>($addParams$io$addArgs)"
           case UniqueAndRawPointers =>
             importListSrc.add("memory")
-            s"std::make_unique<${types2class(t.name)}>($addParams$io$addArgs)"
+            // C++14
+            //s"std::make_unique<${types2class(t.name)}>($addParams$io$addArgs)"
+            s"std::unique_ptr(new ${types2class(t.name)}($addParams$io$addArgs))"
         }
     }
   }

From 627ae6830c0d4e3b678fd9228297b1aaa82769a2 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 25 Dec 2018 10:34:53 +0000
Subject: [PATCH 200/230] CppCompiler: actually fixed std::unique_ptr
 invocation

---
 .../src/main/scala/io/kaitai/struct/languages/CppCompiler.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 26e4e8794..dd9f79a3c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -683,7 +683,7 @@ class CppCompiler(
             importListSrc.add("memory")
             // C++14
             //s"std::make_unique<${types2class(t.name)}>($addParams$io$addArgs)"
-            s"std::unique_ptr(new ${types2class(t.name)}($addParams$io$addArgs))"
+            s"std::unique_ptr<${types2class(t.name)}>(new ${types2class(t.name)}($addParams$io$addArgs))"
         }
     }
   }

From 6fbcf3cb82c6dc590bb2bf0db3a3836f5d41289d Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 25 Dec 2018 17:38:05 +0000
Subject: [PATCH 201/230] DataType: pure data types should be always
 non-owning, so it should be CalcKaitaiStructType

---
 shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index c814b4b89..1d38046b6 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -416,7 +416,7 @@ object DataType {
       )
     case "str" => CalcStrType
     case "bool" => CalcBooleanType
-    case "struct" => KaitaiStructType
+    case "struct" => CalcKaitaiStructType
     case "io" => KaitaiStreamType
     case "any" => AnyType
     case _ => UserTypeInstream(classNameToList(dt), None)

From 8cee2de921bd5a577d3053e785ac498cb5049015 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 27 Dec 2018 06:43:30 +0000
Subject: [PATCH 202/230] DataType: fix user type passing to make user types
 non-owning

---
 shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index 1d38046b6..e7b9b29c3 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -419,7 +419,7 @@ object DataType {
     case "struct" => CalcKaitaiStructType
     case "io" => KaitaiStreamType
     case "any" => AnyType
-    case _ => UserTypeInstream(classNameToList(dt), None)
+    case _ => CalcUserType(classNameToList(dt), None)
   }
 
   def getEncoding(curEncoding: Option[String], metaDef: MetaSpec, path: List[String]): String = {

From fd7e64cf0280da7e76b47f53018de445e17b9177 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 27 Dec 2018 18:06:33 +0000
Subject: [PATCH 203/230] CppCompiler: fix use of std::move in push_back in
 C++11 mode

---
 .../src/main/scala/io/kaitai/struct/languages/CppCompiler.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index dd9f79a3c..4f66cf91c 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -621,7 +621,7 @@ class CppCompiler(
     }
     outSrc.puts(s"$typeDecl$tempVar = $expr;")
 
-    outSrc.puts(s"${privateMemberName(id)}->push_back($tempVar);")
+    outSrc.puts(s"${privateMemberName(id)}->push_back(${stdMoveWrap(tempVar)});")
   }
 
   override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = {

From 910901b345792099bd9f007a92ec08d0044d5cc6 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 27 Dec 2018 18:17:48 +0000
Subject: [PATCH 204/230] CppCompiler: avoid generating get() for switch types
 which result in non-pointer

---
 .../src/main/scala/io/kaitai/struct/languages/CppCompiler.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 4f66cf91c..b3493997b 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -898,6 +898,8 @@ class CppCompiler(
         privateMemberName(attrName)
       case UniqueAndRawPointers =>
         attrType match {
+          case st: SwitchType =>
+            nonOwningPointer(attrName, st.combinedType)
           case t: ComplexDataType =>
             if (t.isOwning) {
               s"${privateMemberName(attrName)}.get()"

From 4312512b138c6b9188b59c3ee61398078079e407 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 10 Jan 2019 01:34:18 +0000
Subject: [PATCH 205/230] GraphvizClassCompiler: moved seqPosToStr into public
 object, adjusted expr => Ast.expr

---
 .../kaitai/struct/GraphvizClassCompiler.scala | 65 ++++++++++---------
 1 file changed, 35 insertions(+), 30 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
index dd2218900..2769d08de 100644
--- a/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/GraphvizClassCompiler.scala
@@ -3,7 +3,6 @@ package io.kaitai.struct
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
-import io.kaitai.struct.exprlang.Ast.expr
 import io.kaitai.struct.format._
 import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic}
 import io.kaitai.struct.precompile.CalculateSeqSizes
@@ -100,18 +99,6 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
     tableEnd
   }
 
-  def seqPosToStr(seqPos: Option[Int]): Option[String] = {
-    seqPos.map { (pos) =>
-      val posByte = pos / 8
-      val posBit = pos % 8
-      if (posBit == 0) {
-        s"$posByte"
-      } else {
-        s"$posByte:$posBit"
-      }
-    }
-  }
-
   def compileParseInstance(className: List[String], id: InstanceIdentifier, inst: ParseInstanceSpec): Unit = {
     val name = id.name
     val lastInstPos = inst.pos
@@ -272,46 +259,46 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
     }
   }
 
-  def expressionSize(ex: expr, attrName: String): String = {
+  def expressionSize(ex: Ast.expr, attrName: String): String = {
     expression(ex, getGraphvizNode(nowClassName, nowClass, attrName) + s":${attrName}_size", STYLE_EDGE_SIZE)
   }
 
-  def expressionPos(ex: expr, attrName: String): String = {
+  def expressionPos(ex: Ast.expr, attrName: String): String = {
     expression(ex, getGraphvizNode(nowClassName, nowClass, attrName) + s":${attrName}_pos", STYLE_EDGE_POS)
   }
 
-  def expressionType(ex: expr, attrName: String): String = {
+  def expressionType(ex: Ast.expr, attrName: String): String = {
     expression(ex, getGraphvizNode(nowClassName, nowClass, attrName) + s":${attrName}_type", STYLE_EDGE_VALUE)
   }
 
-  def expression(e: expr, portName: String, style: String): String = {
+  def expression(e: Ast.expr, portName: String, style: String): String = {
     affectedVars(e).foreach((v) =>
       links += ((v, portName, style))
     )
     htmlEscape(translator.translate(e))
   }
 
-  def affectedVars(e: expr): List[String] = {
+  def affectedVars(e: Ast.expr): List[String] = {
     e match {
-      case expr.BoolOp(op, values) =>
+      case Ast.expr.BoolOp(op, values) =>
         values.flatMap(affectedVars).toList
-      case expr.BinOp(left, op, right) =>
+      case Ast.expr.BinOp(left, op, right) =>
         affectedVars(left) ++ affectedVars(right)
-      case expr.UnaryOp(op, operand) =>
+      case Ast.expr.UnaryOp(op, operand) =>
         affectedVars(operand)
-      case expr.IfExp(condition, ifTrue, ifFalse) =>
+      case Ast.expr.IfExp(condition, ifTrue, ifFalse) =>
         affectedVars(condition) ++ affectedVars(ifTrue) ++ affectedVars(ifFalse)
       //      case expr.Dict(keys, values) =>
-      case expr.Compare(left, ops, right) =>
+      case Ast.expr.Compare(left, ops, right) =>
         affectedVars(left) ++ affectedVars(right)
       //      case expr.Call(func, args) =>
-      case expr.IntNum(_) | expr.FloatNum(_) | expr.Str(_) | expr.Bool(_) =>
+      case Ast.expr.IntNum(_) | Ast.expr.FloatNum(_) | Ast.expr.Str(_) | Ast.expr.Bool(_) =>
         List()
-      case _: expr.EnumByLabel =>
+      case _: Ast.expr.EnumByLabel =>
         List()
-      case expr.EnumById(_, id, _) =>
+      case Ast.expr.EnumById(_, id, _) =>
         affectedVars(id)
-      case expr.Attribute(value, attr) =>
+      case Ast.expr.Attribute(value, attr) =>
         val targetClass = translator.detectType(value)
         targetClass match {
           case t: UserType =>
@@ -328,12 +315,12 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
           case _ =>
             affectedVars(value)
         }
-      case expr.Subscript(value, idx) =>
+      case Ast.expr.Subscript(value, idx) =>
         affectedVars(value) ++ affectedVars(idx)
       case SwitchType.ELSE_CONST =>
         // "_" is a special const for
         List()
-      case expr.Name(id) =>
+      case Ast.expr.Name(id) =>
         if (id.name.charAt(0) == '_') {
           // other special consts like "_io", "_index", etc
           List()
@@ -341,7 +328,7 @@ class GraphvizClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends
           // this must be local name, resolve it
           List(resolveLocalNode(id.name))
         }
-      case expr.List(elts) =>
+      case Ast.expr.List(elts) =>
         elts.flatMap(affectedVars).toList
     }
   }
@@ -447,4 +434,22 @@ object GraphvizClassCompiler extends LanguageCompilerStatic {
   def htmlEscape(s: String): String = {
     s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll("\"", "&quot;")
   }
+
+  /**
+    * Converts bit-level position into byte/bit human-readable combination.
+    * @param seqPos optional number of bits
+    * @return fractional human-readable string which displays "bytes:bits",
+    *         akin to "minutes:seconds" time display
+    */
+  def seqPosToStr(seqPos: Option[Int]): Option[String] = {
+    seqPos.map { (pos) =>
+      val posByte = pos / 8
+      val posBit = pos % 8
+      if (posBit == 0) {
+        s"$posByte"
+      } else {
+        s"$posByte:$posBit"
+      }
+    }
+  }
 }

From 91707e624821566e0c70bc005394b59207356127 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 11 Jan 2019 10:56:26 +0000
Subject: [PATCH 206/230] More unit tests for TypeDetector + SwitchType

---
 .../struct/datatype/SwitchType$Test.scala     | 40 +++++++++++++++++++
 .../translators/TypeDetector$Test.scala       | 16 ++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 jvm/src/test/scala/io/kaitai/struct/datatype/SwitchType$Test.scala
 create mode 100644 jvm/src/test/scala/io/kaitai/struct/translators/TypeDetector$Test.scala

diff --git a/jvm/src/test/scala/io/kaitai/struct/datatype/SwitchType$Test.scala b/jvm/src/test/scala/io/kaitai/struct/datatype/SwitchType$Test.scala
new file mode 100644
index 000000000..eb2fef40f
--- /dev/null
+++ b/jvm/src/test/scala/io/kaitai/struct/datatype/SwitchType$Test.scala
@@ -0,0 +1,40 @@
+package io.kaitai.struct.datatype
+
+import io.kaitai.struct.datatype.DataType.SwitchType
+import io.kaitai.struct.exprlang.Expressions
+import io.kaitai.struct.format.ClassSpec
+import org.scalatest.FunSpec
+import org.scalatest.Matchers._
+
+class SwitchType$Test extends FunSpec {
+  describe("SwitchType.parseSwitch") {
+    it ("combines ints properly") {
+      val t = SwitchType(
+        Expressions.parse("foo"),
+        Map(
+          Expressions.parse("1") -> DataType.IntMultiType(true, DataType.Width2, Some(LittleEndian)),
+          Expressions.parse("2") -> DataType.IntMultiType(false, DataType.Width4, Some(LittleEndian))
+        )
+      )
+
+      t.combinedType should be(DataType.CalcIntType)
+    }
+
+    it ("combines owning user types properly") {
+      val ut1 = DataType.UserTypeInstream(List("foo"), None)
+      ut1.classSpec = Some(ClassSpec.opaquePlaceholder(List("foo")))
+      val ut2 = DataType.UserTypeInstream(List("bar"), None)
+//      ut2.classSpec = Some(ClassSpec.opaquePlaceholder(List("bar")))
+
+      val t = SwitchType(
+        Expressions.parse("foo"),
+        Map(
+          Expressions.parse("1") -> ut1,
+          Expressions.parse("2") -> ut2
+        )
+      )
+
+      t.combinedType should be(DataType.KaitaiStructType)
+    }
+  }
+}
diff --git a/jvm/src/test/scala/io/kaitai/struct/translators/TypeDetector$Test.scala b/jvm/src/test/scala/io/kaitai/struct/translators/TypeDetector$Test.scala
new file mode 100644
index 000000000..1e81a236e
--- /dev/null
+++ b/jvm/src/test/scala/io/kaitai/struct/translators/TypeDetector$Test.scala
@@ -0,0 +1,16 @@
+package io.kaitai.struct.translators
+
+import io.kaitai.struct.datatype.DataType._
+import org.scalatest.FunSpec
+import org.scalatest.Matchers._
+
+class TypeDetector$Test extends FunSpec {
+  describe("TypeDetector") {
+    it("combines ints properly") {
+      val ut1 = CalcUserType(List("foo"), None)
+      val ut2 = CalcUserType(List("bar"), None)
+
+      TypeDetector.combineTypes(ut1, ut2) should be(CalcKaitaiStructType)
+    }
+  }
+}

From ec21bdf62bcc9211e514dd8413ed5f6714a8e6bc Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 13 Jan 2019 00:34:04 +0000
Subject: [PATCH 207/230] Started simple `html` target compiler

---
 .../io/kaitai/struct/DocClassCompiler.scala   |  83 ++++++++++
 .../io/kaitai/struct/HtmlClassCompiler.scala  | 152 ++++++++++++++++++
 .../main/scala/io/kaitai/struct/Main.scala    |   2 +
 .../components/LanguageCompilerStatic.scala   |   2 +-
 4 files changed, 238 insertions(+), 1 deletion(-)
 create mode 100644 shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala
 create mode 100644 shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala

diff --git a/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala
new file mode 100644
index 000000000..af2376516
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala
@@ -0,0 +1,83 @@
+package io.kaitai.struct
+
+import io.kaitai.struct.format._
+import io.kaitai.struct.precompile.CalculateSeqSizes
+import io.kaitai.struct.translators.RubyTranslator
+
+abstract class DocClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends AbstractCompiler {
+  val provider = new ClassTypeProvider(classSpecs, topClass)
+  val translator = new RubyTranslator(provider)
+
+  // TODO: move it into SingleOutputFile equivalent
+  val out = new StringLanguageOutputWriter(indent)
+  def outFileName(topClass: ClassSpec): String
+  def indent: String
+  // END move to SingleOutputFile
+
+  def nowClass: ClassSpec = provider.nowClass
+  def nowClassName = provider.nowClass.name
+
+  override def compile: CompileLog.SpecSuccess = {
+    fileHeader(topClass)
+    compileClass(topClass)
+    fileFooter(topClass)
+
+    CompileLog.SpecSuccess(
+      "",
+      List(CompileLog.FileSuccess(
+        outFileName(topClass),
+        out.result
+      ))
+    )
+  }
+
+  def compileClass(curClass: ClassSpec): Unit = {
+    provider.nowClass = curClass
+    val className = curClass.name
+
+    classHeader(curClass)
+
+    // Sequence
+    compileSeq(curClass)
+
+    curClass.instances.foreach { case (instName, instSpec) =>
+      instSpec match {
+        case pis: ParseInstanceSpec =>
+          compileParseInstance(pis)
+        case vis: ValueInstanceSpec =>
+          compileValueInstance(vis)
+      }
+    }
+
+    curClass.enums.foreach { case(enumName, enumColl) => compileEnum(enumName, enumColl) }
+
+    // Recursive types
+    curClass.types.foreach { case (typeName, intClass) => compileClass(intClass) }
+
+    classFooter(curClass)
+  }
+
+  def compileSeq(curClass: ClassSpec): Unit = {
+    seqHeader(curClass)
+
+    CalculateSeqSizes.forEachSeqAttr(curClass, (attr, seqPos, sizeElement, sizeContainer) => {
+      compileSeqAttr(attr, seqPos, sizeElement, sizeContainer)
+    })
+
+    seqFooter(curClass)
+  }
+
+  def fileHeader(topClass: ClassSpec): Unit
+  def fileFooter(topClass: ClassSpec): Unit
+
+  def classHeader(classSpec: ClassSpec): Unit
+  def classFooter(classSpec: ClassSpec): Unit
+
+  def seqHeader(classSpec: ClassSpec): Unit
+  def seqFooter(classSpec: ClassSpec): Unit
+
+  def compileSeqAttr(attr: AttrSpec, seqPos: Option[Int], sizeElement: Sized, sizeContainer: Sized): Unit
+  def compileParseInstance(inst: ParseInstanceSpec): Unit
+  def compileValueInstance(vis: ValueInstanceSpec): Unit
+  def compileEnum(enumName: String, enumColl: EnumSpec): Unit
+}
diff --git a/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala
new file mode 100644
index 000000000..4405217cc
--- /dev/null
+++ b/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala
@@ -0,0 +1,152 @@
+package io.kaitai.struct
+
+import io.kaitai.struct.datatype.DataType
+import io.kaitai.struct.datatype.DataType.UserType
+import io.kaitai.struct.exprlang.Ast
+import io.kaitai.struct.format._
+import io.kaitai.struct.languages.components.{LanguageCompiler, LanguageCompilerStatic}
+
+class HtmlClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends DocClassCompiler(classSpecs, topClass) {
+  import HtmlClassCompiler._
+
+  override def outFileName(topClass: ClassSpec): String = s"${topClass.nameAsStr}.html"
+
+  override def indent: String = ""
+
+  override def fileHeader(topClass: ClassSpec): Unit = {
+    out.puts(
+      s"""
+        |<!doctype html>
+        |<html lang="en">
+        |  <head>
+        |    <!-- Required meta tags -->
+        |    <meta charset="utf-8">
+        |    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+        |
+        |    <!-- Bootstrap CSS -->
+        |    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/css/bootstrap.min.css" integrity="sha384-GJzZqFGwb1QTTN6wy59ffF1BuGJpLSa9DkKMp0DgiMDm4iYMj70gZWKYbI706tWS" crossorigin="anonymous">
+        |
+        |    <title>${type2str(topClass.name.last)} format specification</title>
+        |  </head>
+        |  <body>
+           <div class="container">
+        |  <h1>${type2str(topClass.name.last)} format specification</h1>
+        |
+      """.stripMargin)
+
+    // TODO: parse & output meta/title, meta/file-extensions, etc
+  }
+
+  override def fileFooter(topClass: ClassSpec): Unit = {
+    out.puts(
+      """
+        |  </div>
+        |    <!-- Optional JavaScript -->
+        |    <!-- jQuery first, then Popper.js, then Bootstrap JS -->
+        |    <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
+        |    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.6/umd/popper.min.js" integrity="sha384-wHAiFfRlMFy6i5SRaxvfOCifBUQy1xHdJ/yoi7FRNXMRBu5WHdZYu1hA6ZOblgut" crossorigin="anonymous"></script>
+        |    <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.2.1/js/bootstrap.min.js" integrity="sha384-B0UglyR+jN6CkvvICOB2joaf5I4l3gm9GU6Hc1og6Ls7i6U/mkkaduKaBhlAXv9k" crossorigin="anonymous"></script>
+        |  </body>
+        |</html>
+      """.stripMargin)
+  }
+
+  override def classHeader(classSpec: ClassSpec): Unit = {
+    out.puts(s"<a name='${classSpec2Anchor(classSpec)}'></a>")
+    out.puts(s"<$headerByIndent>Type: ${type2str(classSpec.name.last)}</$headerByIndent>")
+    out.puts
+
+    classSpec.doc.summary.foreach(summary =>
+      out.puts(s"<p>$summary</p>")
+    )
+    out.inc
+  }
+
+  override def classFooter(classSpec: ClassSpec): Unit = {
+    out.dec
+  }
+
+  override def seqHeader(classSpec: ClassSpec): Unit = {
+    out.puts("<table class=\"table\">")
+    out.puts("<tr><th>Offset</th><th>Size</th><th>ID</th><th>Type</th><th>Note</th></tr>")
+  }
+
+  override def seqFooter(classSpec: ClassSpec): Unit = {
+    out.puts("</table>")
+  }
+
+  override def compileSeqAttr(attr: AttrSpec, seqPos: Option[Int], sizeElement: Sized, sizeContainer: Sized): Unit = {
+    out.puts("<tr>")
+    out.puts(s"<td>${GraphvizClassCompiler.seqPosToStr(seqPos).getOrElse("???")}</td>")
+    out.puts(s"<td>...</td>")
+    out.puts(s"<td>${attr.id.humanReadable}</td>")
+    out.puts(s"<td>${kaitaiType2NativeType(attr.dataType)}</td>")
+    out.puts(s"<td>${attr.doc.summary.getOrElse("")}</td>")
+    out.puts("</tr>")
+  }
+
+  override def compileParseInstance(inst: ParseInstanceSpec): Unit = {
+    out.puts(s"<p><b>Parse instance</b>: ${inst.id.humanReadable}</p>")
+    out.puts("<table class=\"table\">")
+    out.puts("<tr>")
+    out.puts(s"<td>${expression(inst.pos)}</td>")
+    out.puts(s"<td>...</td>")
+    out.puts(s"<td>${inst.id.humanReadable}</td>")
+    out.puts(s"<td>${kaitaiType2NativeType(inst.dataType)}</td>")
+    out.puts(s"<td>${inst.doc.summary.getOrElse("")}</td>")
+    out.puts("</tr>")
+    out.puts("</table>")
+  }
+
+  override def compileValueInstance(vis: ValueInstanceSpec): Unit = {
+    out.puts(s"value instance: ${vis}")
+  }
+
+  override def compileEnum(enumName: String, enumColl: EnumSpec): Unit = {
+    out.puts(s"<a name='${enumSpec2Anchor(enumColl)}'></a>")
+    out.puts(s"<$headerByIndent>Enum: $enumName</$headerByIndent>")
+    out.puts
+
+    out.puts("<table class=\"table\">")
+    out.puts("<tr>")
+    out.puts("<th>ID</th><th>Name</th><th>Note</th>")
+    out.puts("</tr>")
+
+    enumColl.sortedSeq.foreach { case (id, value) =>
+      out.puts("<tr>")
+      out.puts(s"<td>$id</td><td>${value.name}</td><td>${value.doc.summary.getOrElse("")}</td></tr>")
+      out.puts("</tr>")
+    }
+
+    out.puts("</table>")
+  }
+
+  def headerByIndent: String = s"h${out.indentLevel + 1}"
+
+  def expression(exOpt: Option[Ast.expr]): String = {
+    exOpt match {
+      case Some(ex) => translator.translate(ex)
+      case None => ""
+    }
+  }
+}
+
+object HtmlClassCompiler extends LanguageCompilerStatic {
+  // FIXME: Unused, should be probably separated from LanguageCompilerStatic
+  override def getCompiler(
+    tp: ClassTypeProvider,
+    config: RuntimeConfig
+  ): LanguageCompiler = ???
+
+  def type2str(name: String): String = Utils.upperCamelCase(name)
+
+  def classSpec2Anchor(spec: ClassSpec): String = "type-" + spec.name.mkString("-")
+
+  def enumSpec2Anchor(spec: EnumSpec): String = "enum-" + spec.name.mkString("-")
+
+  def kaitaiType2NativeType(attrType: DataType): String = attrType match {
+    case ut: UserType =>
+      "<a href=\"#" + classSpec2Anchor(ut.classSpec.get) + "\">" + type2str(ut.name.last) + "</a>"
+    case _ => GraphvizClassCompiler.dataTypeName(attrType)
+  }
+}
diff --git a/shared/src/main/scala/io/kaitai/struct/Main.scala b/shared/src/main/scala/io/kaitai/struct/Main.scala
index 08e0ca514..15f00b34f 100644
--- a/shared/src/main/scala/io/kaitai/struct/Main.scala
+++ b/shared/src/main/scala/io/kaitai/struct/Main.scala
@@ -63,6 +63,8 @@ object Main {
         new RustClassCompiler(specs, spec, config)
       case ConstructClassCompiler =>
         new ConstructClassCompiler(specs, spec)
+      case HtmlClassCompiler =>
+        new HtmlClassCompiler(specs, spec)
       case _ =>
         new ClassCompiler(specs, spec, config, lang)
     }
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
index 3c8aacacc..bbb497c96 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/components/LanguageCompilerStatic.scala
@@ -2,7 +2,6 @@ package io.kaitai.struct.languages.components
 
 import io.kaitai.struct._
 import io.kaitai.struct.languages._
-import io.kaitai.struct.translators.{BaseTranslator, TypeProvider}
 
 trait LanguageCompilerStatic {
   def getCompiler(tp: ClassTypeProvider, config: RuntimeConfig): LanguageCompiler
@@ -15,6 +14,7 @@ object LanguageCompilerStatic {
     "csharp" -> CSharpCompiler,
     "graphviz" -> GraphvizClassCompiler,
     "go" -> GoCompiler,
+    "html" -> HtmlClassCompiler,
     "java" -> JavaCompiler,
     "javascript" -> JavaScriptCompiler,
     "lua" -> LuaCompiler,

From d4321303fe49dc0393824440b9c99ade6fb767d0 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 18 Jan 2019 19:19:32 +0000
Subject: [PATCH 208/230] ParseInstanceSpec: more specific identifier

---
 .../src/main/scala/io/kaitai/struct/format/InstanceSpec.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
index 2577ac99d..4da5050c8 100644
--- a/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
+++ b/shared/src/main/scala/io/kaitai/struct/format/InstanceSpec.scala
@@ -18,7 +18,7 @@ case class ValueInstanceSpec(
   override def isNullable: Boolean = ifExpr.isDefined
 }
 case class ParseInstanceSpec(
-  id: Identifier,
+  id: InstanceIdentifier,
   path: List[String],
   private val _doc: DocSpec,
   dataType: DataType,

From 5c3018285caa6dba2e3e513b2bdf716669b1de2a Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 21 Jan 2019 17:18:22 +0000
Subject: [PATCH 209/230] DocClassCompiler: added passing of class into
 instance invocations

---
 .../scala/io/kaitai/struct/DocClassCompiler.scala | 15 ++++++++-------
 .../io/kaitai/struct/HtmlClassCompiler.scala      |  4 ++--
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala
index af2376516..685cd1c38 100644
--- a/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/DocClassCompiler.scala
@@ -33,26 +33,27 @@ abstract class DocClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) ext
 
   def compileClass(curClass: ClassSpec): Unit = {
     provider.nowClass = curClass
-    val className = curClass.name
 
     classHeader(curClass)
 
     // Sequence
     compileSeq(curClass)
 
-    curClass.instances.foreach { case (instName, instSpec) =>
+    // Instances
+    curClass.instances.foreach { case (_, instSpec) =>
       instSpec match {
         case pis: ParseInstanceSpec =>
-          compileParseInstance(pis)
+          compileParseInstance(curClass, pis)
         case vis: ValueInstanceSpec =>
           compileValueInstance(vis)
       }
     }
 
+    // Enums
     curClass.enums.foreach { case(enumName, enumColl) => compileEnum(enumName, enumColl) }
 
     // Recursive types
-    curClass.types.foreach { case (typeName, intClass) => compileClass(intClass) }
+    curClass.types.foreach { case (_, intClass) => compileClass(intClass) }
 
     classFooter(curClass)
   }
@@ -61,7 +62,7 @@ abstract class DocClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) ext
     seqHeader(curClass)
 
     CalculateSeqSizes.forEachSeqAttr(curClass, (attr, seqPos, sizeElement, sizeContainer) => {
-      compileSeqAttr(attr, seqPos, sizeElement, sizeContainer)
+      compileSeqAttr(curClass, attr, seqPos, sizeElement, sizeContainer)
     })
 
     seqFooter(curClass)
@@ -76,8 +77,8 @@ abstract class DocClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) ext
   def seqHeader(classSpec: ClassSpec): Unit
   def seqFooter(classSpec: ClassSpec): Unit
 
-  def compileSeqAttr(attr: AttrSpec, seqPos: Option[Int], sizeElement: Sized, sizeContainer: Sized): Unit
-  def compileParseInstance(inst: ParseInstanceSpec): Unit
+  def compileSeqAttr(classSpec: ClassSpec, attr: AttrSpec, seqPos: Option[Int], sizeElement: Sized, sizeContainer: Sized): Unit
+  def compileParseInstance(classSpec: ClassSpec, inst: ParseInstanceSpec): Unit
   def compileValueInstance(vis: ValueInstanceSpec): Unit
   def compileEnum(enumName: String, enumColl: EnumSpec): Unit
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala b/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala
index 4405217cc..9d9d74c1a 100644
--- a/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/HtmlClassCompiler.scala
@@ -75,7 +75,7 @@ class HtmlClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends Doc
     out.puts("</table>")
   }
 
-  override def compileSeqAttr(attr: AttrSpec, seqPos: Option[Int], sizeElement: Sized, sizeContainer: Sized): Unit = {
+  override def compileSeqAttr(classSpec: ClassSpec, attr: AttrSpec, seqPos: Option[Int], sizeElement: Sized, sizeContainer: Sized): Unit = {
     out.puts("<tr>")
     out.puts(s"<td>${GraphvizClassCompiler.seqPosToStr(seqPos).getOrElse("???")}</td>")
     out.puts(s"<td>...</td>")
@@ -85,7 +85,7 @@ class HtmlClassCompiler(classSpecs: ClassSpecs, topClass: ClassSpec) extends Doc
     out.puts("</tr>")
   }
 
-  override def compileParseInstance(inst: ParseInstanceSpec): Unit = {
+  override def compileParseInstance(classSpec: ClassSpec, inst: ParseInstanceSpec): Unit = {
     out.puts(s"<p><b>Parse instance</b>: ${inst.id.humanReadable}</p>")
     out.puts("<table class=\"table\">")
     out.puts("<tr>")

From baed6124e83f8c6f78b90c3ac7eb6e51619ce9d8 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 25 Jan 2019 20:43:19 +0000
Subject: [PATCH 210/230] Output errors where they belong - to stderr

---
 jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
index d41e05838..dcfe0eaa5 100644
--- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
+++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
@@ -346,7 +346,7 @@ class JavaMain(config: CLIConfig) {
 
   private def exceptionToCompileError(ex: Throwable, srcFile: String): CompileError = {
     if (!config.jsonOutput)
-      Console.println(ex.getMessage)
+      Console.err.println(ex.getMessage)
     ex match {
       case ype: YAMLParseException =>
         CompileError("(main)", ype.path, ype.msg)

From 97c928603871651189136a5c96c20f7d534f2a8f Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 25 Jan 2019 21:59:16 +0000
Subject: [PATCH 211/230] PythonCompiler: implement SEQ_FIELDS in `--debug`
 mode

---
 .../scala/io/kaitai/struct/languages/PythonCompiler.scala    | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index bac7a846b..c54e2f9ed 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -450,6 +450,11 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     out.dec
   }
 
+  override def debugClassSequence(seq: List[AttrSpec]) = {
+    val seqStr = seq.map((attr) => "\"" + idToStr(attr.id) + "\"").mkString(", ")
+    out.puts(s"SEQ_FIELDS = [$seqStr]")
+  }
+
   def bool2Py(b: Boolean): String = if (b) { "True" } else { "False" }
 
   def idToStr(id: Identifier): String = {

From 154e2afe12d089b92bccb6a08c1bcc172e5b1f32 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Fri, 25 Jan 2019 22:03:06 +0000
Subject: [PATCH 212/230] Fix circleci test stage

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0c70156df..989939808 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -26,7 +26,7 @@ jobs:
   test:
     <<: *defaults
     steps:
-      - run: cat /dev/null | sbt compilerJVM/test
+      - run: cat /dev/null | sbt test
 workflows:
   version: 2
   build_test_deploy:

From 877563020d3b22f375fce4978d17e480a9983d84 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 3 Mar 2019 09:50:30 +0000
Subject: [PATCH 213/230] JavaCompiler: fix CalcKaitaiStructType handling

---
 .../main/scala/io/kaitai/struct/languages/JavaCompiler.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
index 1de08c004..370b760a6 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/JavaCompiler.scala
@@ -763,7 +763,7 @@ object JavaCompiler extends LanguageCompilerStatic
 
       case AnyType => "Object"
       case KaitaiStreamType => kstreamName
-      case KaitaiStructType => kstructName
+      case KaitaiStructType | CalcKaitaiStructType => kstructName
 
       case t: UserType => types2class(t.name)
       case EnumType(name, _) => types2class(name)

From 03e7af21bccfbdbe26e9733c651e53723fa1da2e Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 3 Mar 2019 09:51:01 +0000
Subject: [PATCH 214/230] DataType: more sealed stuff to reduce warning surface

---
 .../scala/io/kaitai/struct/datatype/DataType.scala     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index e7b9b29c3..b5994e640 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -27,14 +27,14 @@ object DataType {
     * A common trait for all types that can be read with a simple,
     * parameterless KaitaiStream API call.
     */
-  trait ReadableType extends DataType {
+  sealed trait ReadableType extends DataType {
     def apiCall(defEndian: Option[FixedEndian]): String
   }
 
-  abstract class NumericType extends DataType
-  abstract class BooleanType extends DataType
+  abstract sealed class NumericType extends DataType
+  abstract sealed class BooleanType extends DataType
 
-  abstract class IntType extends NumericType
+  abstract sealed class IntType extends NumericType
   case object CalcIntType extends IntType
   case class Int1Type(signed: Boolean) extends IntType with ReadableType {
     override def apiCall(defEndian: Option[FixedEndian]): String = if (signed) "s1" else "u1"
@@ -103,7 +103,7 @@ object DataType {
     * manage their own creation/destruction, borrowed rely on other doing
     * that.
     */
-  abstract class ComplexDataType extends DataType {
+  abstract sealed class ComplexDataType extends DataType {
     /**
       * @return If true, this is "owning" type: for languages where data ownership
       *         matters, this one represents primary owner of the data block, who

From af3e235c38fbc2fb1490f1ed0dade7d9ca7fc522 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 3 Mar 2019 14:20:51 +0000
Subject: [PATCH 215/230] Added extra checks/warnings

---
 .../io/kaitai/struct/precompile/LoadImports.scala      | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/LoadImports.scala b/shared/src/main/scala/io/kaitai/struct/precompile/LoadImports.scala
index 1cb0c0304..bd9684704 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/LoadImports.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/LoadImports.scala
@@ -58,9 +58,18 @@ class LoadImports(specs: ClassSpecs) {
     }
 
     futureSpec.flatMap { case optSpec =>
+      Log.importOps.info(() => {
+        val specNameAsStr = optSpec.map(_.nameAsStr).getOrElse("<none>")
+        s".. LoadImports: loadImport($name, workDir = $workDir), got spec=$specNameAsStr"
+      })
       optSpec match {
         case Some(spec) =>
           val specName = spec.name.head
+          // Check if spec name does not match file name. If it doesn't match,
+          // it is probably already a serious error.
+          if (name != specName)
+            Log.importOps.warn(() => s"... expected to have type name $name, but got $specName")
+
           // Check if we've already had this spec in our ClassSpecs. If we do,
           // don't do anything: we've already processed it and reprocessing it
           // might lead to infinite recursion.
@@ -71,6 +80,7 @@ class LoadImports(specs: ClassSpecs) {
             specs(specName) = spec
             processClass(spec, ImportPath.updateWorkDir(workDir, impPath))
           } else {
+            Log.importOps.warn(() => s"... we have that already, ignoring")
             Future { List() }
           }
         case None =>

From 9c991945ff46f97d818d89b338bebc949ac3aa4b Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 3 Mar 2019 14:22:18 +0000
Subject: [PATCH 216/230] JS main flow: moved part of main flow to
 JavaScriptKSYParser.yamlToSpecs, akin to JavaKSYParser.localFileToSpecs

---
 js/src/main/scala/io/kaitai/struct/MainJs.scala | 13 ++++++-------
 .../struct/format/JavaScriptKSYParser.scala     | 17 +++++++++++++++++
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/js/src/main/scala/io/kaitai/struct/MainJs.scala b/js/src/main/scala/io/kaitai/struct/MainJs.scala
index 342054305..9e11d0ad9 100644
--- a/js/src/main/scala/io/kaitai/struct/MainJs.scala
+++ b/js/src/main/scala/io/kaitai/struct/MainJs.scala
@@ -1,13 +1,13 @@
 package io.kaitai.struct
 
-import io.kaitai.struct.format.{ClassSpec, JavaScriptClassSpecs, JavaScriptKSYParser, KSVersion}
+import io.kaitai.struct.format.{JavaScriptKSYParser, KSVersion}
 import io.kaitai.struct.languages.components.LanguageCompilerStatic
 
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
 import scala.scalajs.js
 import scala.scalajs.js.JSConverters._
 import scala.scalajs.js.annotation.JSExport
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Future
 
 @JSExport
 object MainJs {
@@ -16,13 +16,12 @@ object MainJs {
   @JSExport
   def compile(langStr: String, yaml: js.Object, importer: JavaScriptImporter, debug: Boolean = false): js.Promise[js.Dictionary[String]] = {
     try {
+      // TODO: add proper enabled by a flag
+      //Log.initFromVerboseFlag(Seq("file", "value", "parent", "type_resolve", "type_valid", "seq_sizes", "import"))
       val config = new RuntimeConfig(autoRead = !debug, readStoresPos = debug)
       val lang = LanguageCompilerStatic.byString(langStr)
 
-      val yamlScala = JavaScriptKSYParser.yamlJavascriptToScala(yaml)
-      val firstSpec = ClassSpec.fromYaml(yamlScala)
-      val specs = new JavaScriptClassSpecs(importer, firstSpec)
-      Main.importAndPrecompile(specs, config).map { (_) =>
+      JavaScriptKSYParser.yamlToSpecs(yaml, importer, config).map { (specs) =>
         specs.flatMap({ case (_, spec) =>
           val files = Main.compile(specs, spec, lang, config).files
           files.map((x) => x.fileName -> x.contents).toMap
diff --git a/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala b/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala
index 4aa0e1f29..4aafd4e4d 100644
--- a/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala
+++ b/js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala
@@ -1,8 +1,25 @@
 package io.kaitai.struct.format
 
+import io.kaitai.struct.{JavaScriptImporter, Main, RuntimeConfig}
+
+import scala.concurrent.Future
 import scala.scalajs.js
+import scala.concurrent.ExecutionContext.Implicits.global
 
 object JavaScriptKSYParser {
+  /**
+    * Converts first YAML (given as JavaScript object) to the ClassSpecs
+    * object, fully imported and precompiled.
+    * @param yaml first KSY file (YAML), given as JavaScript object
+    * @return future of ClassSpecs object
+    */
+  def yamlToSpecs(yaml: Any, importer: JavaScriptImporter, config: RuntimeConfig): Future[ClassSpecs] = {
+    val yamlScala = yamlJavascriptToScala(yaml)
+    val firstSpec = ClassSpec.fromYaml(yamlScala)
+    val specs = new JavaScriptClassSpecs(importer, firstSpec)
+    Main.importAndPrecompile(specs, config).map((_) => specs)
+  }
+
   def yamlJavascriptToScala(src: Any): Any = {
     src match {
       case array: js.Array[AnyRef] =>

From 50394015ab013422c83298db118a6c06298dd261 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 3 Mar 2019 23:08:10 +0000
Subject: [PATCH 217/230] PythonTranslator: use safe resolve_enum to resolve
 enums, avoid problems with invalid values

---
 .../io/kaitai/struct/translators/PythonTranslator.scala     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
index 7b48c2908..7f44219de 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/PythonTranslator.scala
@@ -4,7 +4,7 @@ import io.kaitai.struct.{ImportList, Utils}
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
 import io.kaitai.struct.format.Identifier
-import io.kaitai.struct.languages.PythonCompiler
+import io.kaitai.struct.languages.{PythonCompiler, RubyCompiler}
 
 class PythonTranslator(provider: TypeProvider, importList: ImportList) extends BaseTranslator(provider) {
   override def numericBinOp(left: Ast.expr, op: Ast.operator, right: Ast.expr) = {
@@ -54,8 +54,8 @@ class PythonTranslator(provider: TypeProvider, importList: ImportList) extends B
 
   override def doEnumByLabel(enumTypeAbs: List[String], label: String): String =
     s"${PythonCompiler.types2class(enumTypeAbs)}.$label"
-  override def doEnumById(enumTypeAbs: List[String], id: String) =
-    s"${PythonCompiler.types2class(enumTypeAbs)}($id)"
+  override def doEnumById(enumTypeAbs: List[String], id: String): String =
+    s"${PythonCompiler.kstreamName}.resolve_enum(${PythonCompiler.types2class(enumTypeAbs)}, $id)"
 
   override def booleanOp(op: Ast.boolop) = op match {
     case Ast.boolop.Or => "or"

From 1d98e6d35053e1fb47723884e518af1db2cf3ea6 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 10 Mar 2019 12:05:19 +0000
Subject: [PATCH 218/230] LuaCompiler: implement userTypeDebugRead, restore CI
 build

---
 .../main/scala/io/kaitai/struct/languages/LuaCompiler.scala    | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala
index 3330b3c71..a671b90cf 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/LuaCompiler.scala
@@ -357,6 +357,9 @@ class LuaCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     expr2
   }
 
+  override def userTypeDebugRead(id: String): Unit =
+    out.puts(s"$id:_read()")
+
   override def switchStart(id: Identifier, on: Ast.expr): Unit =
     out.puts(s"local _on = ${expression(on)}")
   override def switchCaseFirstStart(condition: Ast.expr): Unit = {

From f20a5afa91fe18b6ac3be99a6161728063277a5b Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 12 Mar 2019 13:04:56 +0000
Subject: [PATCH 219/230] Pumped copyright

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4cca194e2..db2c22b2f 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ If you're looking for information on:
 
 ### Main code
 
-Kaitai Struct compiler itself is copyright (C) 2015-2018 Kaitai
+Kaitai Struct compiler itself is copyright (C) 2015-2019 Kaitai
 Project.
 
 This program is free software: you can redistribute it and/or modify

From 8fd1f5734b38273b32a653e050b0133c91620faf Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Sun, 17 Mar 2019 16:12:28 +0000
Subject: [PATCH 220/230] CppCompiler: work around more C++11 switch type
 getting issues

---
 .../kaitai/struct/languages/CppCompiler.scala | 39 ++++++++++++-------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index b3493997b..71c83bc6d 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -899,7 +899,7 @@ class CppCompiler(
       case UniqueAndRawPointers =>
         attrType match {
           case st: SwitchType =>
-            nonOwningPointer(attrName, st.combinedType)
+            nonOwningPointer(attrName, combineSwitchType(st))
           case t: ComplexDataType =>
             if (t.isOwning) {
               s"${privateMemberName(attrName)}.get()"
@@ -986,19 +986,30 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
       }
 
       case st: SwitchType =>
-        val ct1 = TypeDetector.combineTypes(
-          // C++ does not have a concept of AnyType, and common use case "lots of incompatible UserTypes
-          // for cases + 1 BytesType for else" combined would result in exactly AnyType - so we try extra
-          // hard to avoid that here with this pre-filtering. In C++, "else" case with raw byte array would
-          // be available through _raw_* attribute anyway.
-          st.cases.filterNot { case (caseExpr, caseValue) => caseExpr == SwitchType.ELSE_CONST }.values
-        )
-        val ct2 = if (st.isOwning) {
-          ct1
-        } else {
-          ct1.asNonOwning
-        }
-        kaitaiType2NativeType(config, ct2, absolute)
+        kaitaiType2NativeType(config, combineSwitchType(st), absolute)
+    }
+  }
+
+  /**
+    * C++ does not have a concept of AnyType, and common use case "lots of
+    * incompatible UserTypes for cases + 1 BytesType for else" combined would
+    * result in exactly AnyType - so we try extra hard to avoid that here with
+    * this pre-filtering. In C++, "else" case with raw byte array would
+    * be available through _raw_* attribute anyway.
+    *
+    * @param st switch type to combine into one overall type
+    * @return
+    */
+  def combineSwitchType(st: SwitchType): DataType = {
+    val ct1 = TypeDetector.combineTypes(
+      st.cases.filterNot {
+        case (caseExpr, _) => caseExpr == SwitchType.ELSE_CONST
+      }.values
+    )
+    if (st.isOwning) {
+      ct1
+    } else {
+      ct1.asNonOwning
     }
   }
 

From 835dfdecd6b9e2d76e6f11abd1ed06ad0b618c24 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Mon, 18 Mar 2019 11:59:02 +0000
Subject: [PATCH 221/230] C++: wrapped std::vector in std::unique_ptr,
 implemented casts as raw

---
 .../io/kaitai/struct/datatype/DataType.scala  | 13 +++-
 .../kaitai/struct/languages/CppCompiler.scala | 61 ++++++++++++-------
 .../struct/translators/CppTranslator.scala    |  4 +-
 3 files changed, 50 insertions(+), 28 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
index b5994e640..e23f39658 100644
--- a/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
+++ b/shared/src/main/scala/io/kaitai/struct/datatype/DataType.scala
@@ -93,7 +93,6 @@ object DataType {
   case class StrFromBytesType(bytes: BytesType, encoding: String) extends StrType
 
   case object CalcBooleanType extends BooleanType
-  case class ArrayType(elType: DataType) extends DataType
 
   /**
     * Complex data type is a data type which creation and destruction is
@@ -156,14 +155,22 @@ object DataType {
     bytes: BytesType,
     override val process: Option[ProcessExpr]
   ) extends UserType(_name, _forcedParent, _args) with Processing {
-    def isOwning = true
+    override def isOwning = true
   }
   case class CalcUserType(
     _name: List[String],
     _forcedParent: Option[Ast.expr],
     _args: Seq[Ast.expr] = Seq()
   ) extends UserType(_name, _forcedParent, _args) {
-    def isOwning = false
+    override def isOwning = false
+  }
+
+  case class ArrayType(elType: DataType) extends ComplexDataType {
+    override def isOwning: Boolean = true
+    override def asNonOwning: CalcArrayType = CalcArrayType(elType)
+  }
+  case class CalcArrayType(elType: DataType) extends ComplexDataType {
+    override def isOwning: Boolean = false
   }
 
   val USER_TYPE_NO_PARENT = Ast.expr.Bool(false)
diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index 71c83bc6d..eaf960ac0 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -370,20 +370,20 @@ class CppCompiler(
 
   def destructMember(id: Identifier, innerType: DataType, isArray: Boolean, hasRaw: Boolean, hasIO: Boolean): Unit = {
     if (isArray) {
-      // raw is std::vector<string>*, no need to delete its contents, but we
-      // need to clean up the vector pointer itself
-      if (hasRaw)
-        outSrc.puts(s"delete ${privateMemberName(RawIdentifier(id))};")
-
-      // IO is std::vector<kstream*>*, needs destruction of both members
-      // and the vector pointer itself
-      if (hasIO) {
-        val ioVar = privateMemberName(IoStorageIdentifier(RawIdentifier(id)))
-        destructVector(s"$kstreamName*", ioVar)
-        outSrc.puts(s"delete $ioVar;")
-      }
-
       if (config.cppConfig.pointers == CppRuntimeConfig.RawPointers) {
+        // raw is std::vector<string>*, no need to delete its contents, but we
+        // need to clean up the vector pointer itself
+        if (hasRaw)
+          outSrc.puts(s"delete ${privateMemberName(RawIdentifier(id))};")
+
+        // IO is std::vector<kstream*>*, needs destruction of both members
+        // and the vector pointer itself
+        if (hasIO) {
+          val ioVar = privateMemberName(IoStorageIdentifier(RawIdentifier(id)))
+          destructVector(s"$kstreamName*", ioVar)
+          outSrc.puts(s"delete $ioVar;")
+        }
+
         // main member contents
         if (needsDestruction(innerType)) {
           val arrVar = privateMemberName(id)
@@ -546,10 +546,10 @@ class CppCompiler(
     importListHdr.add("vector")
 
     if (needRaw) {
-      outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = new std::vector<std::string>();")
-      outSrc.puts(s"${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))} = new std::vector<$kstreamName*>();")
+      outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = ${newVector(CalcBytesType)};")
+      outSrc.puts(s"${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))} = ${newVector(KaitaiStreamType)};")
     }
-    outSrc.puts(s"${privateMemberName(id)} = new std::vector<${kaitaiType2NativeType(dataType)}>();")
+    outSrc.puts(s"${privateMemberName(id)} = ${newVector(dataType)};")
     outSrc.puts("{")
     outSrc.inc
     outSrc.puts("int i = 0;")
@@ -576,13 +576,13 @@ class CppCompiler(
     outSrc.puts(s"int $lenVar = ${expression(repeatExpr)};")
     if (needRaw) {
       val rawId = privateMemberName(RawIdentifier(id))
-      outSrc.puts(s"$rawId = new std::vector<std::string>();")
+      outSrc.puts(s"$rawId = ${newVector(CalcBytesType)};")
       outSrc.puts(s"$rawId->reserve($lenVar);")
       val ioId = privateMemberName(IoStorageIdentifier(RawIdentifier(id)))
-      outSrc.puts(s"$ioId = new std::vector<$kstreamName*>();")
+      outSrc.puts(s"$ioId = ${newVector(KaitaiStreamType)};")
       outSrc.puts(s"$ioId->reserve($lenVar);")
     }
-    outSrc.puts(s"${privateMemberName(id)} = new std::vector<${kaitaiType2NativeType(dataType)}>();")
+    outSrc.puts(s"${privateMemberName(id)} = ${newVector(dataType)};")
     outSrc.puts(s"${privateMemberName(id)}->reserve($lenVar);")
     outSrc.puts(s"for (int i = 0; i < $lenVar; i++) {")
     outSrc.inc
@@ -601,10 +601,10 @@ class CppCompiler(
     importListHdr.add("vector")
 
     if (needRaw) {
-      outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = new std::vector<std::string>();")
-      outSrc.puts(s"${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))} = new std::vector<$kstreamName*>();")
+      outSrc.puts(s"${privateMemberName(RawIdentifier(id))} = ${newVector(CalcBytesType)};")
+      outSrc.puts(s"${privateMemberName(IoStorageIdentifier(RawIdentifier(id)))} = ${newVector(KaitaiStreamType)};")
     }
-    outSrc.puts(s"${privateMemberName(id)} = new std::vector<${kaitaiType2NativeType(dataType)}>();")
+    outSrc.puts(s"${privateMemberName(id)} = ${newVector(dataType)};")
     outSrc.puts("{")
     outSrc.inc
     outSrc.puts("int i = 0;")
@@ -688,6 +688,17 @@ class CppCompiler(
     }
   }
 
+  def newVector(elType: DataType): String = {
+    val cppElType = kaitaiType2NativeType(elType)
+    config.cppConfig.pointers match {
+      case RawPointers =>
+        s"new std::vector<$cppElType>()"
+      case UniqueAndRawPointers =>
+        s"std::unique_ptr<std::vector<$cppElType>>(new std::vector<$cppElType>())"
+        // TODO: C++14 with std::make_unique
+    }
+  }
+
   override def bytesPadTermExpr(expr0: String, padRight: Option[Int], terminator: Option[Int], include: Boolean) = {
     val expr1 = padRight match {
       case Some(padByte) => s"$kstreamName::bytes_strip_right($expr0, $padByte)"
@@ -971,7 +982,11 @@ object CppCompiler extends LanguageCompilerStatic with StreamStructNames {
           t.name
         })
 
-      case ArrayType(inType) => s"std::vector<${kaitaiType2NativeType(config, inType, absolute)}>*"
+      case ArrayType(inType) => config.pointers match {
+        case RawPointers => s"std::vector<${kaitaiType2NativeType(config, inType, absolute)}>*"
+        case UniqueAndRawPointers => s"std::unique_ptr<std::vector<${kaitaiType2NativeType(config, inType, absolute)}>>"
+      }
+      case CalcArrayType(inType) => s"std::vector<${kaitaiType2NativeType(config, inType, absolute)}>*"
 
       case KaitaiStreamType => s"$kstreamName*"
       case KaitaiStructType => config.pointers match {
diff --git a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
index 59e1b66f4..fb4b64e37 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/CppTranslator.scala
@@ -2,7 +2,7 @@ package io.kaitai.struct.translators
 
 import java.nio.charset.Charset
 
-import io.kaitai.struct.CppRuntimeConfig.{RawPointers, SharedPointers}
+import io.kaitai.struct.CppRuntimeConfig.{RawPointers, SharedPointers, UniqueAndRawPointers}
 import io.kaitai.struct.datatype.DataType
 import io.kaitai.struct.datatype.DataType._
 import io.kaitai.struct.exprlang.Ast
@@ -149,7 +149,7 @@ class CppTranslator(provider: TypeProvider, importListSrc: ImportList, config: R
     s"((${translate(condition)}) ? (${translate(ifTrue)}) : (${translate(ifFalse)}))"
   override def doCast(value: Ast.expr, typeName: DataType): String =
     config.cppConfig.pointers match {
-      case RawPointers =>
+      case RawPointers | UniqueAndRawPointers =>
         cppStaticCast(value, typeName)
       case SharedPointers =>
         typeName match {

From 1816e97be8008aa580cf5b96b1ecd29cd273ef94 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Tue, 19 Mar 2019 02:36:44 +0000
Subject: [PATCH 222/230] CppCompiler: repeat-until fixes for C++11

---
 .../kaitai/struct/languages/CppCompiler.scala | 20 ++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
index eaf960ac0..93a3884cb 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/CppCompiler.scala
@@ -608,20 +608,34 @@ class CppCompiler(
     outSrc.puts("{")
     outSrc.inc
     outSrc.puts("int i = 0;")
-    outSrc.puts(s"${kaitaiType2NativeType(dataType)} ${translator.doName("_")};")
+    outSrc.puts(s"${kaitaiType2NativeType(dataType.asNonOwning)} ${translator.doName("_")};")
     outSrc.puts("do {")
     outSrc.inc
   }
 
+  private val ReStdUniquePtr = "^std::unique_ptr<(.*?)>\\((.*?)\\)$".r
+
   override def handleAssignmentRepeatUntil(id: Identifier, expr: String, isRaw: Boolean): Unit = {
     val (typeDecl, tempVar) = if (isRaw) {
       ("std::string ", translator.doName(Identifier.ITERATOR2))
     } else {
       ("", translator.doName(Identifier.ITERATOR))
     }
-    outSrc.puts(s"$typeDecl$tempVar = $expr;")
 
-    outSrc.puts(s"${privateMemberName(id)}->push_back(${stdMoveWrap(tempVar)});")
+    val (wrappedTempVar, rawPtrExpr) = if (config.cppConfig.pointers == UniqueAndRawPointers) {
+      expr match {
+        case ReStdUniquePtr(cppClass, innerExpr) =>
+          (s"std::move(std::unique_ptr<$cppClass>($tempVar))", innerExpr)
+        case _ =>
+          (tempVar, expr)
+      }
+    } else {
+      (tempVar, expr)
+    }
+
+    outSrc.puts(s"$typeDecl$tempVar = $rawPtrExpr;")
+
+    outSrc.puts(s"${privateMemberName(id)}->push_back($wrappedTempVar);")
   }
 
   override def condRepeatUntilFooter(id: Identifier, io: String, dataType: DataType, needRaw: Boolean, untilExpr: expr): Unit = {

From c5639b74acc855e1c8077a04fdd884b3f21a4488 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 20 Mar 2019 07:56:35 +0000
Subject: [PATCH 223/230] PythonCompiler: use imported classes without `from
 ... import ...`, keeping them in their module namespaces => works around
 circular imports problem

---
 .../struct/languages/PythonCompiler.scala     | 26 ++++++++++++++-----
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index c54e2f9ed..37a262032 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -66,12 +66,13 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = {
     val name = classSpec.name.head
-    val prefix = config.pythonPackage match {
-      case "" => ""
-      case "." => "."
-      case pkg => s"$pkg."
-    }
-    out.puts(s"from $prefix$name import ${type2class(name)}")
+    out.puts(s"import $opaquePrefix$name")
+  }
+
+  private def opaquePrefix = config.pythonPackage match {
+    case "" => ""
+    case "." => "."
+    case pkg => s"$pkg."
   }
 
   override def classHeader(name: String): Unit = {
@@ -377,7 +378,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
           }
           s", $parent, self._root$addEndian"
         }
-        s"${types2class(t.classSpec.get.name)}($addParams$io$addArgs)"
+        s"${userType2class(t)}($addParams$io$addArgs)"
     }
   }
 
@@ -479,6 +480,17 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
   }
 
   override def localTemporaryName(id: Identifier): String = s"_t_${idToStr(id)}"
+
+  def userType2class(t: UserType): String = {
+    val name = t.classSpec.get.name
+    val firstName = name.head
+    val prefix = if (t.isOpaque && firstName != translator.provider.nowClass.name.head) {
+      s"$opaquePrefix$firstName."
+    } else {
+      ""
+    }
+    s"$prefix${types2class(name)}"
+  }
 }
 
 object PythonCompiler extends LanguageCompilerStatic

From d097cc5b36bc4437b3fb3b4b4871775376e66ad6 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Thu, 21 Mar 2019 00:40:10 +0000
Subject: [PATCH 224/230] PythonCompiler: fix config.pythonPackage handling to
 properly address modules in all cases

---
 .../kaitai/struct/languages/PythonCompiler.scala | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
index 37a262032..4ca11c961 100644
--- a/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
+++ b/shared/src/main/scala/io/kaitai/struct/languages/PythonCompiler.scala
@@ -66,13 +66,13 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
 
   override def opaqueClassDeclaration(classSpec: ClassSpec): Unit = {
     val name = classSpec.name.head
-    out.puts(s"import $opaquePrefix$name")
-  }
-
-  private def opaquePrefix = config.pythonPackage match {
-    case "" => ""
-    case "." => "."
-    case pkg => s"$pkg."
+    out.puts(
+      if (config.pythonPackage.nonEmpty) {
+        s"from ${config.pythonPackage} import $name"
+      } else {
+        s"import $name"
+      }
+    )
   }
 
   override def classHeader(name: String): Unit = {
@@ -485,7 +485,7 @@ class PythonCompiler(typeProvider: ClassTypeProvider, config: RuntimeConfig)
     val name = t.classSpec.get.name
     val firstName = name.head
     val prefix = if (t.isOpaque && firstName != translator.provider.nowClass.name.head) {
-      s"$opaquePrefix$firstName."
+      s"$firstName."
     } else {
       ""
     }

From 69960862e1d530aef82e944f74c4daeefd6889e0 Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 3 Apr 2019 16:42:45 +0100
Subject: [PATCH 225/230] Added two tests to assess "not..." operation parsing

---
 .../kaitai/struct/exprlang/ExpressionsSpec.scala   | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
index 1ac1ecb9c..338be47a5 100644
--- a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
@@ -202,6 +202,20 @@ class ExpressionsSpec extends FunSpec {
       Expressions.parse("truer") should be (Name(identifier("truer")))
     }
 
+    // Boolean operations
+    it("parses not foo") {
+      Expressions.parse("not foo") should be (
+        UnaryOp(
+          Ast.unaryop.Not,
+          Name(identifier("foo"))
+        )
+      )
+    }
+
+    it("parses note_len") {
+      Expressions.parse("note_len") should be (Name(identifier("note_len")))
+    }
+
     // String literals
     it("parses simple string") {
       Expressions.parse("\"abc\"") should be (Str("abc"))

From 0221a5a436974736f54ab1517bf5620614973d4c Mon Sep 17 00:00:00 2001
From: Mikhail Yakshin <greycat@altlinux.org>
Date: Wed, 3 Apr 2019 16:56:52 +0100
Subject: [PATCH 226/230] Fixed "not" parsing bug + added 2 more related tests

---
 .../kaitai/struct/exprlang/ExpressionsSpec.scala | 16 ++++++++++++++++
 .../io/kaitai/struct/exprlang/Expressions.scala  |  2 +-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
index 338be47a5..0c4769671 100644
--- a/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
+++ b/jvm/src/test/scala/io/kaitai/struct/exprlang/ExpressionsSpec.scala
@@ -216,6 +216,22 @@ class ExpressionsSpec extends FunSpec {
       Expressions.parse("note_len") should be (Name(identifier("note_len")))
     }
 
+    it("parses notnot") {
+      Expressions.parse("notnot") should be (Name(identifier("notnot")))
+    }
+
+    it("parses not not true") {
+      Expressions.parse("not not true") should be (
+        UnaryOp(
+          Ast.unaryop.Not,
+          UnaryOp(
+            Ast.unaryop.Not,
+            Bool(true)
+          )
+        )
+      )
+    }
+
     // String literals
     it("parses simple string") {
       Expressions.parse("\"abc\"") should be (Str("abc"))
diff --git a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
index bc80f454c..405de889d 100644
--- a/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
+++ b/shared/src/main/scala/io/kaitai/struct/exprlang/Expressions.scala
@@ -47,7 +47,7 @@ object Expressions {
     case Seq(x) => x
     case xs => Ast.expr.BoolOp(Ast.boolop.And, xs)
   }
-  val not_test: P[Ast.expr] = P( ("not" ~ not_test).map(Ast.expr.UnaryOp(Ast.unaryop.Not, _)) | comparison )
+  val not_test: P[Ast.expr] = P( (kw("not") ~ not_test).map(Ast.expr.UnaryOp(Ast.unaryop.Not, _)) | comparison )
   val comparison: P[Ast.expr] = P( expr ~ (comp_op ~ expr).? ).map{
     case (lhs, None) => lhs
     case (lhs, Some(chunks)) =>

From 5c561bd405b48235922cb2834c68cf568e9aee61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <6223655+ams-tschoening@users.noreply.github.com>
Date: Thu, 4 Apr 2019 20:00:29 +0200
Subject: [PATCH 227/230] Allow "import-path" more than once to not necessarily
 use OS-specific path separator. This fixes #557.

---
 jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
index dcfe0eaa5..cb0a1eb29 100644
--- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
+++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
@@ -62,7 +62,7 @@ object JavaMain {
       } text("output directory (filenames will be auto-generated)")
 
       val importPathExample = List("<directory>", "<directory>", "...").mkString(File.pathSeparator)
-      opt[String]('I', "import-path") valueName(importPathExample) action { (x, c) =>
+      opt[String]('I', "import-path") optional() unbounded() valueName(importPathExample) action { (x, c) =>
         c.copy(importPaths = c.importPaths ++ x.split(File.pathSeparatorChar))
       } text(".ksy library search path(s) for imports (see also KSPATH env variable)")
 

From a7d5e44f57d0517fcc490d208219abb71c2f5066 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <6223655+ams-tschoening@users.noreply.github.com>
Date: Thu, 4 Apr 2019 20:00:29 +0200
Subject: [PATCH 228/230] Allow "import-path" more than once to not necessarily
 use OS-specific path separator. This fixes #557.

---
 jvm/src/main/scala/io/kaitai/struct/JavaMain.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
index dcfe0eaa5..cb0a1eb29 100644
--- a/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
+++ b/jvm/src/main/scala/io/kaitai/struct/JavaMain.scala
@@ -62,7 +62,7 @@ object JavaMain {
       } text("output directory (filenames will be auto-generated)")
 
       val importPathExample = List("<directory>", "<directory>", "...").mkString(File.pathSeparator)
-      opt[String]('I', "import-path") valueName(importPathExample) action { (x, c) =>
+      opt[String]('I', "import-path") optional() unbounded() valueName(importPathExample) action { (x, c) =>
         c.copy(importPaths = c.importPaths ++ x.split(File.pathSeparatorChar))
       } text(".ksy library search path(s) for imports (see also KSPATH env variable)")
 

From f621628fbe7240aead3aca12d0aeb2124aa66693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <6223655+ams-tschoening@users.noreply.github.com>
Date: Fri, 5 Apr 2019 10:38:00 +0200
Subject: [PATCH 229/230] Added some logging for resolveEnumSpec like was the
 case for user types already.

---
 .../src/main/scala/io/kaitai/struct/Log.scala |  5 ++++-
 .../struct/precompile/ResolveTypes.scala      | 19 ++++++++++++-------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/Log.scala b/shared/src/main/scala/io/kaitai/struct/Log.scala
index 1aeb161f6..aef4e1b93 100644
--- a/shared/src/main/scala/io/kaitai/struct/Log.scala
+++ b/shared/src/main/scala/io/kaitai/struct/Log.scala
@@ -27,7 +27,8 @@ object Log {
     "type_resolve",
     "type_valid",
     "seq_sizes",
-    "import"
+    "import",
+    "enum_resolve"
   )
 
   var fileOps: Logger = NullLogger
@@ -37,6 +38,7 @@ object Log {
   var typeValid: Logger = NullLogger
   var seqSizes: Logger = NullLogger
   var importOps: Logger = NullLogger
+  var enumResolve: Logger = NullLogger
 
   def initFromVerboseFlag(subsystems: Seq[String]): Unit = {
     fileOps = NullLogger
@@ -50,6 +52,7 @@ object Log {
       case "type_valid" => typeValid = ConsoleLogger
       case "seq_sizes" => seqSizes = ConsoleLogger
       case "import" => importOps = ConsoleLogger
+      case "enum_resolve" => enumResolve = ConsoleLogger
     }
   }
 }
diff --git a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
index d221c1890..179f46813 100644
--- a/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
+++ b/shared/src/main/scala/io/kaitai/struct/precompile/ResolveTypes.scala
@@ -122,14 +122,19 @@ class ResolveTypes(specs: ClassSpecs, opaqueTypes: Boolean) {
   }
 
   def resolveEnumSpec(curClass: ClassSpec, typeName: List[String]): Option[EnumSpec] = {
-    //    Console.println(s"resolveEnumSpec: at ${curClass.name} doing ${typeName.mkString("|")}")
-    val res = realResolveEnumSpec(curClass, typeName)
-    //    Console.println("   => " + (res match {
-    //      case None => "???"
-    //      case Some(x) => x.name.mkString("|")
-    //    }))
+    Log.enumResolve.info(() => s"resolveEnumSpec: at ${curClass.name} doing ${typeName.mkString("|")}")
 
-    res
+    val res = realResolveEnumSpec(curClass, typeName)
+    res match {
+      case None => {
+        Log.enumResolve.info(() => s"    => ???")
+        res
+      }
+      case Some(x) => {
+        Log.enumResolve.info(() => s"    => ${x.nameAsStr}")
+        res
+      }
+    }
   }
 
   private def realResolveEnumSpec(curClass: ClassSpec, typeName: List[String]): Option[EnumSpec] = {

From c027309f7093bf3e1daf0023780040f37545d533 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Thorsten=20Sch=C3=B6ning?=
 <6223655+ams-tschoening@users.noreply.github.com>
Date: Fri, 5 Apr 2019 17:13:50 +0200
Subject: [PATCH 230/230] Added "enumInverseMap" following "enumDirectMap".

"enumToInt" didn't take the fully qualified path of an ENUM into account when rendering usage of the associated inverse map. If that map needed to be accessed in some external KSY compared to where the map is hosted, things failed because the map was wrongly assumed to be in the KSY where it is used. This commit changes "I__ENUM" to "Class::I__ENUM" following what has been available already as "enumDirectMap".

This fixes https://github.com/kaitai-io/kaitai_struct/issues/552.
---
 .../io/kaitai/struct/translators/RubyTranslator.scala | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
index 3896864ee..c2ee9a100 100644
--- a/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
+++ b/shared/src/main/scala/io/kaitai/struct/translators/RubyTranslator.scala
@@ -56,6 +56,15 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     }
   }
 
+  def enumInverseMap(et: EnumType): String = {
+    val enumTypeAndName = et.enumSpec.get.name
+    val enumDirectMap = this.enumDirectMap(enumTypeAndName)
+    val enumNameDirect = enumTypeAndName.last.toUpperCase
+    val enumNameInverse = RubyCompiler.inverseEnumName(enumNameDirect)
+
+    enumDirectMap.replace(enumNameDirect, enumNameInverse)
+  }
+
   override def doSubscript(container: Ast.expr, idx: Ast.expr): String =
     s"${translate(container)}[${translate(idx)}]"
   override def doIfExp(condition: Ast.expr, ifTrue: Ast.expr, ifFalse: Ast.expr): String =
@@ -70,7 +79,7 @@ class RubyTranslator(provider: TypeProvider) extends BaseTranslator(provider) {
     })
   }
   override def enumToInt(v: Ast.expr, et: EnumType): String =
-    s"${RubyCompiler.inverseEnumName(et.name.last.toUpperCase)}[${translate(v)}]"
+    s"${enumInverseMap(et)}[${translate(v)}]"
   override def floatToInt(v: Ast.expr): String =
     s"(${translate(v)}).to_i"
   override def intToStr(i: Ast.expr, base: Ast.expr): String =