Merge branch 'ams_bug_2543' into calcinttype_int_to_long

.circleci/config.yml

@@ -26,7 +26,7 @@ jobs:
   test:
     <<: *defaults
     steps:
-      - run: cat /dev/null | sbt compilerJVM/test
+      - run: cat /dev/null | sbt test
 workflows:
   version: 2
   build_test_deploy:

README.md

@@ -6,7 +6,7 @@
 
 # Kaitai Struct: compiler
 
-This project is an official reference compiler for [Kaitai Struct](https://github.com/kaitai-io/kaitai_struct) project.
+This project is an official reference compiler for [Kaitai Struct](http://kaitai.io) project.
 
 Kaitai Struct is a declarative language used to describe various
 binary data structures, laid out in files or in memory: i.e. binary
@@ -19,115 +19,22 @@ languages. These modules will include the generated code for a parser
 that can read described data structure from a file / stream and give
 access to it in a nice, easy-to-comprehend API.
 
-Please refer to [documentation in Kaitai Struct project](https://github.com/kaitai-io/kaitai_struct)
-for details on `.ksy` files and general usage patterns.
+## Further information
 
-## Trying without install
+If you're looking for information on:
 
-Kaitai Struct compiler can be tried instantly, without any downloads
-and installation, at
-
-http://kaitai.io/repl
-
-Note that this implementation uses the same reference code as in this
-repository and executes totally on a client side, without any queries
-to server backend.
-
-## Downloading and installing
-
-### Linux .deb builds (Debian/Ubuntu)
-
-There is an official .deb repository available. The repository is hosted
-at BinTray and signed with BinTray GPG key (`379CE192D401AB61`), so it's
-necessary to import that key first if your box haven't used any BinTray
-repositories beforehand:
-
-```shell
-echo "deb https://dl.bintray.com/kaitai-io/debian jessie main" | sudo tee /etc/apt/sources.list.d/kaitai.list
-sudo apt-key adv --keyserver hkp://pool.sks-keyservers.net --recv 379CE192D401AB61
-sudo apt-get update
-sudo apt-get install kaitai-struct-compiler
-```
-
-### Windows builds
-
-An official `.msi` installer build is available to download at
-
-https://bintray.com/kaitai-io/universal/kaitai-struct-compiler/_latestVersion
-
-### Universal builds
-
-Basically, everything that can run Java can use so called "universal"
-builds: a .zip file that includes all the required .jar files bundled
-and launcher scripts for UNIX/Windows systems. No installation
-required, one can just unpack and run it. Available at download also at
-
-https://bintray.com/kaitai-io/universal/kaitai-struct-compiler/_latestVersion
-
-### Source code
-
-If you're interested in developing compiler itself, you can check out
-source code in repository:
-
-    git clone https://github.com/kaitai-io/kaitai_struct_compiler
-
-See the [developer documentation](http://doc.kaitai.io/developers.html) for 
-general pointers on how to proceed with the source code then.
-
-## Usage
-
-`kaitai-struct-compiler [options] <file>...`
-
-Alternatively, a symlink `ksc` is provided and can be used everywhere
-just as full name.
-
-Common options:
-
-* `<file>...` — source files (.ksy)
-* `-t <language> | --target <language>` — target languages (`graphviz`, `csharp`,
-    `all`, `perl`, `java`, `go`, `cpp_stl`, `php`, `lua`, `python`, `ruby`, `javascript`
-  * `all` is a special case: it compiles all possible target
-    languages, creating language-specific directories (as per language
-    identifiers) inside output directory, and then creating output
-    module(s) for each language starting from there
-* `-d <directory> | --outdir <directory>` — output directory
-  (filenames will be auto-generated)
-
-Language-specific options:
-
-* `--dot-net-namespace <namespace>` — .NET namespace (C# only, default: Kaitai)
-* `--java-package <package>` — Java package (Java only, default: root package)
-* `--php-namespace <namespace>` — PHP namespace (PHP only, default: root package)
-
-Misc options:
-
-* `--verbose` — verbose output
-* `--help` — display usage information and exit
-* `--version` — output version information and exit
-
-A few examples, given that file `foo.ksy` exists in current directory
-and describes format with ID `foo`:
-
-* `kaitai-struct-compiler -t python foo.ksy` — compile format in
-  `foo.ksy`, write output in current directory to file `foo.py`
-* `kaitai-struct-compiler -t java foo.ksy` — compile format in
-  `foo.ksy`, create "src" subdir in current one and write output in
-  `src/Foo.java`
-* `kaitai-struct-compiler -t java --java-package org.example foo.ksy`
-  — compile format in `foo.ksy`, create "src/org/example" subdir tree
-  in current one and write output in `src/org/example/Foo.java`;
-  resulting file will bear correct Java package clause.
-* `kaitai-struct-compiler -t all -d /tmp/out --java-package org.example foo.ksy`
-  — compile format in `foo.ksy`, creating a hierarchy of files:
-  * `/tmp/out/java/src/org/example/Foo.java`
-  * `/tmp/out/python/foo.py`
-  * `/tmp/out/ruby/foo.rb`
+* Kaitai Struct language itself (`.ksy` files, general usage patterns)
+  — refer to the [user guide](http://doc.kaitai.io/user_guide.html).
+* How to download and install Kaitai Struct — see the
+  [downloads](http://kaitai.io/#download).
+* How to build the compiler, run the test suite, and join the
+  development — see the [developer memo](http://doc.kaitai.io/developers.html).
 
 ## Licensing
 
 ### Main code
 
-Kaitai Struct compiler itself is copyright (C) 2015-2018 Kaitai
+Kaitai Struct compiler itself is copyright (C) 2015-2019 Kaitai
 Project.
 
 This program is free software: you can redistribute it and/or modify

RELEASE_NOTES.md

@@ -1,4 +1,12 @@
-# 0.8 (TBD)
+# 0.9 (TBD)
+
+* New targets support:
+  * Python with [Construct library](https://construct.readthedocs.io)
+* Expression language:
+  * New methods:
+    * byte arrays: `length`
+
+# 0.8 (2018-02-05)
 
 * New target languages:
   * Lua (96% tests pass score)

build.sbt

@@ -5,7 +5,7 @@ import sbt.Keys._
 
 resolvers += Resolver.sonatypeRepo("public")
 
-val VERSION = "0.8"
+val VERSION = "0.9-SNAPSHOT"
 val TARGET_LANGS = "C++/STL, C#, Java, JavaScript, Lua, Perl, PHP, Python, Ruby"
 
 lazy val root = project.in(file(".")).
@@ -128,6 +128,8 @@ lazy val compiler = crossProject.in(file(".")).
     // https://github.com/sbt/sbt-native-packager/issues/1067
     debianNativeBuildOptions in Debian := Seq("-Zgzip", "-z3"),
 
+    debianPackageDependencies := Seq("java8-runtime-headless"),
+
     packageSummary in Linux := s"compiler to generate binary data parsers in $TARGET_LANGS",
     packageSummary in Windows := "Kaitai Struct compiler",
     packageDescription in Linux :=

js/src/main/scala/io/kaitai/struct/MainJs.scala

@@ -1,13 +1,13 @@
 package io.kaitai.struct
 
-import io.kaitai.struct.format.{ClassSpec, JavaScriptClassSpecs, JavaScriptKSYParser, KSVersion}
+import io.kaitai.struct.format.{JavaScriptKSYParser, KSVersion}
 import io.kaitai.struct.languages.components.LanguageCompilerStatic
 
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
 import scala.scalajs.js
 import scala.scalajs.js.JSConverters._
 import scala.scalajs.js.annotation.JSExport
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.Future
 
 @JSExport
 object MainJs {
@@ -16,13 +16,12 @@ object MainJs {
   @JSExport
   def compile(langStr: String, yaml: js.Object, importer: JavaScriptImporter, debug: Boolean = false): js.Promise[js.Dictionary[String]] = {
     try {
-      val config = new RuntimeConfig(debug = debug)
+      // TODO: add proper enabled by a flag
+      //Log.initFromVerboseFlag(Seq("file", "value", "parent", "type_resolve", "type_valid", "seq_sizes", "import"))
+      val config = new RuntimeConfig(autoRead = !debug, readStoresPos = debug)
       val lang = LanguageCompilerStatic.byString(langStr)
 
-      val yamlScala = JavaScriptKSYParser.yamlJavascriptToScala(yaml)
-      val firstSpec = ClassSpec.fromYaml(yamlScala)
-      val specs = new JavaScriptClassSpecs(importer, firstSpec)
-      Main.importAndPrecompile(specs, config).map { (_) =>
+      JavaScriptKSYParser.yamlToSpecs(yaml, importer, config).map { (specs) =>
         specs.flatMap({ case (_, spec) =>
           val files = Main.compile(specs, spec, lang, config).files
           files.map((x) => x.fileName -> x.contents).toMap

js/src/main/scala/io/kaitai/struct/format/JavaScriptKSYParser.scala

@@ -1,8 +1,25 @@
 package io.kaitai.struct.format
 
+import io.kaitai.struct.{JavaScriptImporter, Main, RuntimeConfig}
+
+import scala.concurrent.Future
 import scala.scalajs.js
+import scala.concurrent.ExecutionContext.Implicits.global
 
 object JavaScriptKSYParser {
+  /**
+    * Converts first YAML (given as JavaScript object) to the ClassSpecs
+    * object, fully imported and precompiled.
+    * @param yaml first KSY file (YAML), given as JavaScript object
+    * @return future of ClassSpecs object
+    */
+  def yamlToSpecs(yaml: Any, importer: JavaScriptImporter, config: RuntimeConfig): Future[ClassSpecs] = {
+    val yamlScala = yamlJavascriptToScala(yaml)
+    val firstSpec = ClassSpec.fromYaml(yamlScala)
+    val specs = new JavaScriptClassSpecs(importer, firstSpec)
+    Main.importAndPrecompile(specs, config).map((_) => specs)
+  }
+
   def yamlJavascriptToScala(src: Any): Any = {
     src match {
       case array: js.Array[AnyRef] =>

jvm/src/main/scala/io/kaitai/struct/JavaMain.scala

@@ -7,6 +7,7 @@ import io.kaitai.struct.CompileLog._
 import io.kaitai.struct.JavaMain.CLIConfig
 import io.kaitai.struct.format.{ClassSpec, ClassSpecs, KSVersion, YAMLParseException}
 import io.kaitai.struct.formats.JavaKSYParser
+import io.kaitai.struct.languages.CppCompiler
 import io.kaitai.struct.languages.components.LanguageCompilerStatic
 import io.kaitai.struct.precompile.ErrorInInput
 
@@ -24,8 +25,9 @@ object JavaMain {
     runtime: RuntimeConfig = RuntimeConfig()
   )
 
-  val ALL_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet
-  val VALID_LANGS = ALL_LANGS + "all"
+  val ALL_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet - "cpp_stl" + "cpp_stl_98" + "cpp_stl_11"
+  val VALID_LANGS = LanguageCompilerStatic.NAME_TO_CLASS.keySet + "all"
+  val CPP_STANDARDS = Set("98", "11")
 
   def parseCommandLine(args: Array[String]): Option[CLIConfig] = {
     val parser = new scopt.OptionParser[CLIConfig](BuildInfo.name) {
@@ -60,10 +62,37 @@ object JavaMain {
       } text("output directory (filenames will be auto-generated)")
 
       val importPathExample = List("<directory>", "<directory>", "...").mkString(File.pathSeparator)
-      opt[String]('I', "import-path") valueName(importPathExample) action { (x, c) =>
+      opt[String]('I', "import-path") optional() unbounded() valueName(importPathExample) action { (x, c) =>
         c.copy(importPaths = c.importPaths ++ x.split(File.pathSeparatorChar))
       } text(".ksy library search path(s) for imports (see also KSPATH env variable)")
 
+      opt[String]("cpp-namespace") valueName("<namespace>") action { (x, c) =>
+        c.copy(
+          runtime = c.runtime.copy(
+            cppConfig = c.runtime.cppConfig.copy(
+              namespace = x.split("::").toList
+            )
+          )
+        )
+      } text("C++ namespace (C++ only, default: none)")
+
+      opt[String]("cpp-standard") valueName("<standard>") action { (x, c) =>
+        c.copy(
+          runtime = c.runtime.copy(
+            cppConfig = x match {
+              case "98" => c.runtime.cppConfig.copyAsCpp98()
+              case "11" => c.runtime.cppConfig.copyAsCpp11()
+            }
+          )
+        )
+      } text("C++ standard to target (C++ only, supported: 98, 11, default: 98)") validate { x =>
+        if (CPP_STANDARDS.contains(x)) {
+          success
+        } else {
+          failure(s"'$x' is not a valid C++ standard to target; valid ones are: ${CPP_STANDARDS.mkString(", ")}")
+        }
+      }
+
       opt[String]("go-package") valueName("<package>") action { (x, c) =>
         c.copy(runtime = c.runtime.copy(goPackage = x))
       } text("Go package (Go only, default: none)")
@@ -115,9 +144,18 @@ object JavaMain {
         }
       }
 
+      opt[Unit]("no-auto-read") action { (x, c) =>
+        c.copy(runtime = c.runtime.copy(autoRead = false))
+      } text("disable auto-running `_read` in constructor")
+
+      opt[Unit]("read-pos") action { (x, c) =>
+        c.copy(runtime = c.runtime.copy(readStoresPos = true))
+      } text("`_read` remembers attribute positions in stream")
+
       opt[Unit]("debug") action { (x, c) =>
-        c.copy(runtime = c.runtime.copy(debug = true))
-      } text("enable debugging helpers (mostly used by visualization tools)")
+        c.copy(runtime = c.runtime.copy(autoRead = false, readStoresPos = true))
+      } text("same as --no-auto-read --read-pos (useful for visualization tools)")
+
       help("help") text("display this help and exit")
       version("version") text("output version information and exit")
     }
@@ -259,10 +297,19 @@ class JavaMain(config: CLIConfig) {
 
   def compileOneLang(specs: ClassSpecs, langStr: String, outDir: String): Map[String, SpecEntry] = {
     Log.fileOps.info(() => s"... compiling it for $langStr... ")
-    val lang = LanguageCompilerStatic.byString(langStr)
+
+    val (lang, fixedRuntime) = langStr match {
+      case "cpp_stl_98" =>
+        (CppCompiler, config.runtime.copy(cppConfig = config.runtime.cppConfig.copyAsCpp98()))
+      case "cpp_stl_11" =>
+        (CppCompiler, config.runtime.copy(cppConfig = config.runtime.cppConfig.copyAsCpp11()))
+      case _ =>
+        (LanguageCompilerStatic.byString(langStr), config.runtime)
+    }
+
     specs.map { case (_, classSpec) =>
       val res = try {
-        compileSpecAndWriteToFile(specs, classSpec, lang, outDir)
+        compileSpecAndWriteToFile(specs, classSpec, lang, fixedRuntime, outDir)
       } catch {
         case ex: Throwable =>
           if (config.throwExceptions)
@@ -277,9 +324,10 @@ class JavaMain(config: CLIConfig) {
     specs: ClassSpecs,
     spec: ClassSpec,
     lang: LanguageCompilerStatic,
+    runtime: RuntimeConfig,
     outDir: String
   ): SpecSuccess = {
-    val res = Main.compile(specs, spec, lang, config.runtime)
+    val res = Main.compile(specs, spec, lang, runtime)
     res.files.foreach { (file) =>
       Log.fileOps.info(() => s".... writing ${file.fileName}")
 
@@ -298,7 +346,7 @@ class JavaMain(config: CLIConfig) {
 
   private def exceptionToCompileError(ex: Throwable, srcFile: String): CompileError = {
     if (!config.jsonOutput)
-      Console.println(ex.getMessage)
+      Console.err.println(ex.getMessage)
     ex match {
       case ype: YAMLParseException =>
         CompileError("(main)", ype.path, ype.msg)

jvm/src/main/scala/io/kaitai/struct/formats/JavaKSYParser.scala

@@ -1,6 +1,7 @@
 package io.kaitai.struct.formats
 
-import java.io.{File, FileReader}
+import java.io._
+import java.nio.charset.Charset
 import java.util.{List => JList, Map => JMap}
 
 import io.kaitai.struct.JavaMain.CLIConfig
@@ -25,11 +26,19 @@ object JavaKSYParser {
 
   def fileNameToSpec(yamlFilename: String): ClassSpec = {
     Log.fileOps.info(() => s"reading $yamlFilename...")
-    val scalaSrc = readerToYaml(new FileReader(yamlFilename))
+
+    // This complex string of classes is due to the fact that Java's
+    // default "FileReader" implementation always uses system locale,
+    // which screws up encoding on some systems and screws up reading
+    // UTF-8 files with BOM
+    val fis = new FileInputStream(yamlFilename)
+    val isr = new InputStreamReader(fis, Charset.forName("UTF-8"))
+    val br = new BufferedReader(isr)
+    val scalaSrc = readerToYaml(br)
     ClassSpec.fromYaml(scalaSrc)
   }
 
-  def readerToYaml(reader: FileReader): Any = {
+  def readerToYaml(reader: Reader): Any = {
     val yamlLoader = new Yaml(new SafeConstructor)
     val javaSrc = yamlLoader.load(reader)
     yamlJavaToScala(javaSrc)