Issue #1036, #1121: CodeGen step

Codegen/src/Generators/DefaultGenerator.cs

@@ -106,6 +106,7 @@ private SourceText LinearGeneration<A>(LinearNodeSourceCodeMapper mapper, IReadO
             Lines_73_80_Flags = new HashSet<int>();
             //The previous line generation buffer 
             StringSourceText previousBuffer = null;
+            bool insideMultilineComments = false;
             for (int i = 0; i < mapper.LineData.Length; i++)
             {
                 if (i == TypeCobolVersionLineNumber && this.TypeCobolVersion != null)
@@ -128,6 +129,19 @@ private SourceText LinearGeneration<A>(LinearNodeSourceCodeMapper mapper, IReadO
                             previousBuffer = null;
                         }
                         string text = Input[i].Text;
+
+                        // Comment the multilines comments
+
+                        if (text.Trim().StartsWith("*<<"))
+                        {
+                            insideMultilineComments = true;
+                        }
+
+                        if (text.Length >= 7 && insideMultilineComments)
+                        {
+                            text = text.Substring(0, 6) + '*' + text.Substring(7, text.Length - 7);
+                        }
+
                         if (mapper.LineData[i].Buffer != null)
                         {
                             //This line has been assigned a target Buffer
@@ -140,6 +154,11 @@ private SourceText LinearGeneration<A>(LinearNodeSourceCodeMapper mapper, IReadO
                             targetSourceText.Insert(text, targetSourceText.Size, targetSourceText.Size);
                             targetSourceText.Insert(Environment.NewLine, targetSourceText.Size, targetSourceText.Size);
                         }
+
+                        if (text.Trim().EndsWith("*>>"))
+                        {
+                            insideMultilineComments = false;
+                        }
                     }
                     continue;
                 }

Codegen/src/Generators/DocumentationGenerator.cs

@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Xml.Serialization;
+using TypeCobol.Compiler;
+using TypeCobol.Compiler.CodeElements;
+using TypeCobol.Compiler.CodeModel;
+using TypeCobol.Compiler.Diagnostics;
+using TypeCobol.Compiler.Nodes;
+using TypeCobol.Compiler.Text;
+
+namespace TypeCobol.Codegen.Generators
+{
+    /// <summary>
+    /// Generator than create the documentation of all applicable Nodes
+    /// </summary>
+    class DocumentationGenerator : IGenerator
+    {
+        private StringBuilder Destination { get; set; }
+        public DocumentationGenerator(StringBuilder destination, string typeCobolVersion)
+        {
+            this.Destination = destination;
+            TypeCobolVersion = typeCobolVersion;
+        }
+        public void Generate(CompilationUnit compilationUnit, ColumnsLayout columns = ColumnsLayout.FreeTextFormat)
+        {
+            Destination.Append("");
+            //Add version to output file
+            if (!string.IsNullOrEmpty(TypeCobolVersion))
+                Destination.AppendLine("      *TypeCobol_Version:" + TypeCobolVersion);
+
+            var sourceFile = compilationUnit.ProgramClassDocumentSnapshot.Root;
+            var docBuilder = new DocumentationBuilder();
+            sourceFile.AcceptASTVisitor(docBuilder);
+
+            // For now serialize as JSON
+            foreach (var doc in docBuilder.DTOs)
+            {
+                Destination.Append(doc.SerializeToJson());
+            }
+            // this.Destination contains now the Json Documentation
+        }
+
+        public List<Diagnostic> Diagnostics { get; set; }
+        public string TypeCobolVersion { get; set; }
+    }
+
+    class DocumentationBuilder : AbstractAstVisitor
+    {
+        public List<Documentation> DTOs;
+
+        public DocumentationBuilder()
+        {
+            DTOs = new List<Documentation>();
+        }
+        public override bool Visit(TypeDefinition node)
+        {
+            DataTypeDescriptionEntry ce = node.CodeElement as DataTypeDescriptionEntry;
+            if (ce != null && ce.Visibility == AccessModifier.Public)
+                DTOs.Add(new DocumentationForType(node));
+            return true;
+        }
+        public override bool Visit(FunctionDeclaration node)
+        {
+            FunctionDeclarationHeader ce = node.CodeElement as FunctionDeclarationHeader;
+            if (ce != null && ce.Visibility == AccessModifier.Public)
+                DTOs.Add(new DocumentationForFunction(node));
+            return true;
+        }
+        public override bool Visit(Program node)
+        {
+            DTOs.Add(new DocumentationForProgram(node));
+            return true;
+        }
+
+    }
+}

Codegen/src/Generators/LinearNodeSourceCodeMapper.cs

@@ -1127,6 +1127,9 @@ public void Accept(Node node)
             //Create All SourceTextBuffer Content associated to Nodes
             CreateNodeSourceTextBufferContents();
 
+            // Comment specific parts (Formalized Comments)
+            CommentSpecificParts(node);
+
             //Now Complete Function Declaration Lines relocation.
             CompleteFunctionDeclarationLinesRelocation();
 
@@ -1281,6 +1284,80 @@ public static bool HasIntersection<A>(List<A> l1, List<A> l2)
             return false;
         }
 
+        /// <summary>
+        /// Return a Tuple of the two tokens specified by the TokenTypes in parameters if any
+        /// </summary>
+        public IEnumerable<Tuple<Token, Token>> GetEnclosingTokensIfAny(Node node, TokenType startTokenType, TokenType stopTokenType)
+        {
+            var startToken =
+                node.CodeElement?.ConsumedTokens.Where(t => t.TokenType == startTokenType).ToArray();
+            var stopToken =
+                node.CodeElement?.ConsumedTokens.Where(t => t.TokenType == stopTokenType).ToArray();
+            if (startToken != null && startToken.Any() &&
+                stopToken != null && stopToken.Any())
+            {
+                // Iterate on the smallest count of token in case of count inequality
+                for (int i = 0; i < (startToken.Count() < stopToken.Count() ? startToken.Count() : stopToken.Count()); i++)
+                {
+                    yield return new Tuple<Token, Token>(startToken[i], stopToken[i]);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Comment a node buffer between two specified tokens
+        /// </summary>
+        public void CommentBetweenTokens(Node node, TokenType startTokenType, TokenType stopTokenType)
+        {
+            // get the NodeData to have the Buffer
+            NodeData nodeData = Nodes.FirstOrDefault(n => n.node == node);
+            if (nodeData != null)
+            {
+                // get the array delimiter consumed Tokens
+                var delimiterTokens = GetEnclosingTokensIfAny(node, startTokenType, stopTokenType)?.ToArray();
+                if (delimiterTokens != null && delimiterTokens.Any())
+                {
+                    int[,] bufferLines = nodeData.Buffer.ComputeLinePositions(0, nodeData.Buffer.Size);
+                    var positionList = nodeData.Buffer.GetPositionList();
+                    foreach (var delimiterToken in delimiterTokens)
+                    {
+
+                        // for each lines, if it is inside the formalized comment then comment them (replace the 7th character by a '*') 
+                        for (int i = 0; i < bufferLines.GetLength(0) - 1; i++)
+                        {
+                            if (nodeData.Positions.Item4[i] >= delimiterToken.Item1.Line &&
+                                nodeData.Positions.Item4[i] <= delimiterToken.Item2.Line &&
+                                bufferLines[i, 1] >= 7) // Avoid comment blank lines
+                            {
+                                if (positionList.Any() && positionList[0].Pos == bufferLines[i, 0] + 6)
+                                {
+                                    positionList[0].SetFlag(Position.Flags.InclStart);
+                                }
+                                nodeData.Buffer.Insert("*", bufferLines[i, 0] + 6, bufferLines[i, 0] + 7);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        public void CommentSpecificParts(Node node)
+        {
+            // Formalised Comments of Programs (Formalized Comments of Typedef and Functions is already commented)
+            if (node is Compiler.Nodes.ProcedureDivision && node.Parent is Compiler.CodeModel.Program)
+            {
+                CommentBetweenTokens(node, TokenType.FORMALIZED_COMMENTS_START, TokenType.FORMALIZED_COMMENTS_STOP);
+            }
+
+            // Multilines Comments
+            CommentBetweenTokens(node, TokenType.MULTILINES_COMMENTS_START, TokenType.MULTILINES_COMMENTS_STOP);
+
+            foreach (var child in node.Children)
+            {
+                CommentSpecificParts(child);
+            }
+        }
+
         /// <summary>
         /// Dump All Structures.
         /// </summary>

Codegen/src/Skeletons/Templates/RazorEngine.cs

@@ -1,4 +1,6 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.Text;
 using RazorEngine;
 using RazorEngine.Templating; // For extension methods.
 
@@ -47,9 +49,40 @@ public string Replace() {
 		public string Replace(string template, Dictionary<string,object> variables = null, string delimiter = null) {
 			if ("@".Equals(delimiter)) throw new System.ArgumentException("Illegal delimiter: @");
 			if (delimiter == null) delimiter = DEFAULT_DELIMITER;
+
+            // get all indexes of the formalized comment start "*<<"
+            List<int> formComStartIOndexes = new List<int>();
+            for (int index = 0; ; index += 2)
+            {
+                index = template.IndexOf("<<", index, StringComparison.Ordinal);
+                if (index != -1)
+                    formComStartIOndexes.Add(index);
+                else
+                    break;
+            }
+
+            // get all indexes of the formalized comment stop "*>>"
+            List<int> formComStopIOndexes = new List<int>();
+            for (int index = 0; ; index += 2)
+            {
+                index = template.IndexOf(">>", index, StringComparison.Ordinal);
+                if (index != -1)
+                    formComStopIOndexes.Add(index);
+                else
+                    break;
+            }
+
+            // escape the '@' inside the formalized comments
+            StringBuilder replacementSB = new StringBuilder(template);
+            for (int i = 0; i < (formComStartIOndexes.Count < formComStopIOndexes.Count ? formComStartIOndexes.Count : formComStopIOndexes.Count); i++)
+            {
+                replacementSB.Replace("@", "@@", formComStartIOndexes[i], formComStopIOndexes[i]);
+            }
+            template = replacementSB.ToString();
+
             //Replace all variable prefix (delimier) by @Model
             //The @Model will contain all the data of the current page (that all substitution environment variables).
-			template = template.Replace(delimiter, "@Model.");
+            template = template.Replace(delimiter, "@Model.");
             //Dynamically create @Model
 			object model = variables == null ? new { } : CreateAnonymousObjectFromDictionary(variables);
 			// TODO: make key unique

Codegen/test/TestTypeCobolCodegen.cs

@@ -788,15 +788,33 @@ public void DeclarativesWithProcedures2()
 	        CodegenTestUtils.ParseGenerateCompare(Path.Combine("TypeCobol", "DeclarativesWithProcedures2") + ".tcbl", skeletons, false, "TestTypeCobolVersion");
 	    }
 
-        [TestMethod]
-        [TestCategory("Codegen")]
-        [TestProperty("Time", "fast")]
-        public void DeclarativesWithInstructionsWithinTest()
-        {
-            var skeletons = CodegenTestUtils.ParseConfig(Path.Combine("TypeCobol", "skeletons") + ".xml");
-            CodegenTestUtils.ParseGenerateCompare(Path.Combine("TypeCobol", "DeclarativesWithInstructionsWithin") + ".rdz.tcbl", skeletons, false, "TestTypeCobolVersion");
+	    [TestMethod]
+	    [TestCategory("Codegen")]
+	    [TestProperty("Time", "fast")]
+	    public void DeclarativesWithInstructionsWithinTest()
+	    {
+	        var skeletons = CodegenTestUtils.ParseConfig(Path.Combine("TypeCobol", "skeletons") + ".xml");
+	        CodegenTestUtils.ParseGenerateCompare(Path.Combine("TypeCobol", "DeclarativesWithInstructionsWithin") + ".rdz.tcbl", skeletons, false, "TestTypeCobolVersion");
         }
 
+	    [TestMethod]
+	    [TestCategory("Codegen")]
+	    [TestProperty("Time", "fast")]
+	    public void FormalizedCommentsTest()
+	    {
+	        var skeletons = CodegenTestUtils.ParseConfig(Path.Combine("TypeCobol", "skeletons") + ".xml");
+	        CodegenTestUtils.ParseGenerateCompare(Path.Combine("TypeCobol", "FormalizedComments") + ".tcbl", skeletons, false, "TestTypeCobolVersion");
+	    }
+
+	    [TestMethod]
+	    [TestCategory("Codegen")]
+	    [TestProperty("Time", "fast")]
+	    public void MultilinesCommentsTest()
+	    {
+	        var skeletons = CodegenTestUtils.ParseConfig(Path.Combine("TypeCobol", "skeletons") + ".xml");
+	        CodegenTestUtils.ParseGenerateCompare(Path.Combine("TypeCobol", "MultilinesComments") + ".tcbl", skeletons, false, "TestTypeCobolVersion");
+	    }
+
 	    [TestMethod]
 	    [TestCategory("Codegen")]
 	    [TestProperty("Time", "fast")]

Codegen/test/resources/input/TypeCobol/FormalizedComments.tcbl

@@ -0,0 +1,61 @@
+ ?       IDENTIFICATION DIVISION.
+        PROGRAM-ID. DocCodeGen.
+
+        DATA DIVISION.
+        LOCAL-STORAGE SECTION.
+
+       *<<< 
+           @ description : inline typedef
+           @ params:
+             - none: Type doesn't have any params
+           @deprec
+           @Restriction : Do not Use BOOL var 
+       *>>>
+        01 myType TYPEDEF STRICT PUBLIC pic X(01).
+
+       *<<< Vect2D
+       *>>>
+        01 Vect2D TYPEDEF STRICT.
+          02 Coord2d.
+            03 X PIC 9(4).
+            03 Y PIC 9(4).
+
+       *<<< My program 
+           @ Description : description
+           @deprecated:
+           @    replacedBy : MyFonction2
+           @ rEsTrIcTiOn : Do not Use BOOL var
+           @ need : some needs
+               - description
+           @ see : Thank you for your attention
+           @ todo :
+               - Add BOOL support 
+               - implement a call counter
+       *>>>
+        PROCEDURE DIVISION.
+
+       *<<< MyProc info
+           @ deprec : It is
+        deprecated
+                @           need : long need
+           @ todo:
+               - todo1
+               - todo 2
+         @ params:
+           - myDate: just a date
+           - bla: bla < 2
+           - toto: toto 
+           -blu: will be ignored
+       *>>>
+        DECLARE PROCEDURE MyProc PUBLIC
+            INPUT    myDate      TYPE Date
+                     bla         Pic S9(1)V9(12)
+            IN-OUT   myBool      TYPE BOOL
+            OUTPUT   toto        TYPE BOOL
+                     bli         Pic PPP999PPP.
+            PROCEDURE DIVISION.
+            CONTINUE.
+        END-DECLARE.
+
+        END PROGRAM DocCodeGen. 
+