witx: Parse comments into documentation in AST (#140)

* witx: add support for parsing multiple top-level witx files

* tests: test parsing multiple top-level witx files

* witx: add a multimodule test that rejects redefined names

and rename the path to be a litle shorter

* witx: add `CommentSyntax` and `Documented` parsers

Derived from ca26654f7747e2c9ce9d80d05c23a4ed48ec126c in
https://github.com/alexcrichton/wat (PR #26)

* witx: Parse comments of top level definitions, render crude markdown

* filter for doc comments, trim whitespace

* some progress on doc comments. wip

* more wip!

* complete adding doc parsing to AST

* bugfixes, use a subcommand to emit docs

* witx: forgot to add docs to module. add docs to output for roundtrip testing

* wasi documents: convert all comments to doc-comments

sed -i -E "s/ ;; / ;;; /" **/*.witx
sed -i -E "s/^;; /;;; /" **/*.witx

* fix fd_advise doc comments to come before parameters

* tests: try to narrow failure a bit

* witx files: normalize leading whitespace in doc-comments

this was stopping the roundtripping test from passing, since the
whitespace in the AST is slightly different after a roundtrip. I am not
feeling clever enough to think of an elegant solution now, so I'll
just hack up these files to not hit the bug.

* witx files: very top comment is not a doc

tools/witx/src/ast.rs

@@ -120,6 +120,7 @@ pub enum DatatypeIdent {
 pub struct Datatype {
     pub name: Id,
     pub variant: DatatypeVariant,
+    pub docs: String,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -149,14 +150,26 @@ pub enum IntRepr {
 pub struct EnumDatatype {
     pub name: Id,
     pub repr: IntRepr,
-    pub variants: Vec<Id>,
+    pub variants: Vec<EnumVariant>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct EnumVariant {
+    pub name: Id,
+    pub docs: String,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct FlagsDatatype {
     pub name: Id,
     pub repr: IntRepr,
-    pub flags: Vec<Id>,
+    pub flags: Vec<FlagsMember>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FlagsMember {
+    pub name: Id,
+    pub docs: String,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -169,6 +182,7 @@ pub struct StructDatatype {
 pub struct StructMember {
     pub name: Id,
     pub type_: DatatypeIdent,
+    pub docs: String,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -181,25 +195,29 @@ pub struct UnionDatatype {
 pub struct UnionVariant {
     pub name: Id,
     pub type_: DatatypeIdent,
+    pub docs: String,
 }
 
 #[derive(Debug, Clone)]
 pub struct Module {
     pub name: Id,
     definitions: Vec<ModuleDefinition>,
     entries: HashMap<Id, ModuleEntry>,
+    pub docs: String,
 }
 
 impl Module {
     pub(crate) fn new(
         name: Id,
         definitions: Vec<ModuleDefinition>,
         entries: HashMap<Id, ModuleEntry>,
+        docs: String,
     ) -> Self {
         Module {
             name,
             definitions,
             entries,
+            docs,
         }
     }
     pub fn import(&self, name: &Id) -> Option<Rc<ModuleImport>> {
@@ -275,6 +293,7 @@ impl PartialEq for ModuleEntry {
 pub struct ModuleImport {
     pub name: Id,
     pub variant: ModuleImportVariant,
+    pub docs: String,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -287,13 +306,15 @@ pub struct InterfaceFunc {
     pub name: Id,
     pub params: Vec<InterfaceFuncParam>,
     pub results: Vec<InterfaceFuncParam>,
+    pub docs: String,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct InterfaceFuncParam {
     pub name: Id,
     pub type_: DatatypeIdent,
     pub position: InterfaceFuncParamPosition,
+    pub docs: String,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]

tools/witx/src/docs.rs

@@ -0,0 +1,227 @@
+use crate::ast::*;
+
+pub trait Documentation {
+    fn to_md(&self) -> String;
+}
+
+impl Documentation for Document {
+    fn to_md(&self) -> String {
+        let mut ret = "# Types\n".to_string();
+        for d in self.datatypes() {
+            ret += &d.to_md();
+        }
+
+        ret += "\n# Modules\n";
+        for m in self.modules() {
+            ret += &m.to_md();
+        }
+        ret
+    }
+}
+
+impl BuiltinType {
+    fn name(&self) -> &'static str {
+        match self {
+            BuiltinType::String => "string",
+            BuiltinType::U8 => "u8",
+            BuiltinType::U16 => "u16",
+            BuiltinType::U32 => "u32",
+            BuiltinType::U64 => "u64",
+            BuiltinType::S8 => "s8",
+            BuiltinType::S16 => "s16",
+            BuiltinType::S32 => "s32",
+            BuiltinType::S64 => "s64",
+            BuiltinType::F32 => "f32",
+            BuiltinType::F64 => "f64",
+        }
+    }
+}
+
+impl DatatypeIdent {
+    fn name(&self) -> String {
+        match self {
+            DatatypeIdent::Builtin(b) => b.name().to_string(),
+            DatatypeIdent::Array(a) => format!("Array<{}>", a.name()),
+            DatatypeIdent::Pointer(p) => format!("Pointer<{}>", p.name()),
+            DatatypeIdent::ConstPointer(p) => format!("ConstPointer<{}>", p.name()),
+            DatatypeIdent::Ident(i) => i.name.as_str().to_string(),
+        }
+    }
+}
+
+impl Documentation for Datatype {
+    fn to_md(&self) -> String {
+        format!(
+            "## `{}`\n{}\n{}\n",
+            self.name.as_str(),
+            self.docs,
+            self.variant.to_md()
+        )
+    }
+}
+
+impl Documentation for DatatypeVariant {
+    fn to_md(&self) -> String {
+        match self {
+            DatatypeVariant::Alias(a) => a.to_md(),
+            DatatypeVariant::Enum(a) => a.to_md(),
+            DatatypeVariant::Flags(a) => a.to_md(),
+            DatatypeVariant::Struct(a) => a.to_md(),
+            DatatypeVariant::Union(a) => a.to_md(),
+        }
+    }
+}
+
+impl Documentation for AliasDatatype {
+    fn to_md(&self) -> String {
+        format!("Alias to `{}`", self.to.name())
+    }
+}
+
+impl Documentation for EnumDatatype {
+    fn to_md(&self) -> String {
+        let variants = self
+            .variants
+            .iter()
+            .map(|v| format!("#### `{}`\n{}", v.name.as_str(), v.docs))
+            .collect::<Vec<String>>()
+            .join("\n");
+        format!(
+            "Enum represented by `{}`\n\n### Variants:\n{}\n",
+            self.repr.name(),
+            variants
+        )
+    }
+}
+
+impl Documentation for FlagsDatatype {
+    fn to_md(&self) -> String {
+        let flags = self
+            .flags
+            .iter()
+            .map(|f| format!("#### `{}`\n{}\n", f.name.as_str(), f.docs))
+            .collect::<Vec<String>>()
+            .join("\n");
+        format!(
+            "Flags represented by `{}`\n\n### Flags:\n{}",
+            self.repr.name(),
+            flags
+        )
+    }
+}
+
+impl Documentation for StructDatatype {
+    fn to_md(&self) -> String {
+        let members = self
+            .members
+            .iter()
+            .map(|m| {
+                format!(
+                    "#### `{}`\nMember type: `{}`\n{}",
+                    m.name.as_str(),
+                    m.type_.name(),
+                    m.docs,
+                )
+            })
+            .collect::<Vec<String>>()
+            .join("\n");
+        format!("### Struct members:\n{}", members)
+    }
+}
+
+impl Documentation for UnionDatatype {
+    fn to_md(&self) -> String {
+        let variants = self
+            .variants
+            .iter()
+            .map(|v| {
+                format!(
+                    "#### `{}`\nVariant type: `{}`\n{}",
+                    v.name.as_str(),
+                    v.type_.name(),
+                    v.docs,
+                )
+            })
+            .collect::<Vec<String>>()
+            .join("\n");
+        format!("### Union variants:\n{}\n", variants)
+    }
+}
+
+impl IntRepr {
+    fn name(&self) -> &'static str {
+        match self {
+            IntRepr::U8 => "u8",
+            IntRepr::U16 => "u16",
+            IntRepr::U32 => "u32",
+            IntRepr::U64 => "u64",
+        }
+    }
+}
+
+impl Documentation for Module {
+    fn to_md(&self) -> String {
+        let imports = self
+            .imports()
+            .map(|i| i.to_md())
+            .collect::<Vec<String>>()
+            .join("\n");
+        let funcs = self
+            .funcs()
+            .map(|i| i.to_md())
+            .collect::<Vec<String>>()
+            .join("\n");
+        format!(
+            "## `{}`\n### Imports\n{}\n### Functions\n{}",
+            self.name.as_str(),
+            imports,
+            funcs,
+        )
+    }
+}
+
+impl Documentation for ModuleImport {
+    fn to_md(&self) -> String {
+        match self.variant {
+            ModuleImportVariant::Memory => format!("* {}: Memory", self.name.as_str()),
+        }
+    }
+}
+
+impl Documentation for InterfaceFunc {
+    fn to_md(&self) -> String {
+        let params = self
+            .params
+            .iter()
+            .map(|f| {
+                format!(
+                    "##### `{name}`\n`{name}` has type `{type_}`\n{docs}",
+                    name = f.name.as_str(),
+                    type_ = f.type_.name(),
+                    docs = f.docs
+                )
+            })
+            .collect::<Vec<String>>()
+            .join("\n");
+        let results = self
+            .results
+            .iter()
+            .map(|f| {
+                format!(
+                    "##### `{name}`\n`{name}` has type `{type_}`\n{docs}",
+                    name = f.name.as_str(),
+                    type_ = f.type_.name(),
+                    docs = f.docs
+                )
+            })
+            .collect::<Vec<String>>()
+            .join("\n");
+        format!(
+            "### {}\n{}\n#### Parameters:\n{}#### Results:\n{}",
+            self.name.as_str(),
+            self.docs,
+            params,
+            results,
+        )
+    }
+}

tools/witx/src/lib.rs

@@ -2,6 +2,8 @@
 mod ast;
 /// Map witx types to core (wasm standard) types
 mod coretypes;
+/// Render documentation
+mod docs;
 /// Interface for filesystem or mock IO
 mod io;
 /// Witx syntax parsing from SExprs
@@ -15,11 +17,12 @@ mod validate;
 
 pub use ast::{
     AliasDatatype, BuiltinType, Datatype, DatatypeIdent, DatatypeVariant, Definition, Document,
-    Entry, EnumDatatype, FlagsDatatype, Id, IntRepr, InterfaceFunc, InterfaceFuncParam,
-    InterfaceFuncParamPosition, Module, ModuleDefinition, ModuleEntry, ModuleImport,
-    ModuleImportVariant, StructDatatype, StructMember, UnionDatatype, UnionVariant,
+    Entry, EnumDatatype, EnumVariant, FlagsDatatype, FlagsMember, Id, IntRepr, InterfaceFunc,
+    InterfaceFuncParam, InterfaceFuncParamPosition, Module, ModuleDefinition, ModuleEntry,
+    ModuleImport, ModuleImportVariant, StructDatatype, StructMember, UnionDatatype, UnionVariant,
 };
 pub use coretypes::{AtomType, CoreFuncType, CoreParamSignifies, CoreParamType, DatatypePassedBy};
+pub use docs::Documentation;
 pub use io::{Filesystem, MockFs, WitxIo};
 pub use parser::DeclSyntax;
 pub use render::{Render, SExpr as RenderSExpr};