Doctest comment parsing using module annotations in Eval Plugin (#1232)

* WIP: Comment parsing using module annotations

* Line Comment parsers (wip)

* Line comment implemented (block comment not implemented)

* Completely switches to Megaparsec

* T27 must be fixed

* We can always assume that comment starts with "--" with no space prepended

* must be horizontal space, not ANY whitespace

* Block parser (WIP)

* We don't need whole range; position suffices

* Brutal parsing for block haddock comments

* Brutal line parsing

* unset Opt_Haddock

* Wrong debug messages

* Redundant debug output

* Hacks for indentation levels and LHS

* Updates block comment logic in Literate Haskell

* Updates doctests

* Allows doctest without newline at the end

* Precise handling of line ending

* Corrects last-line block eval handling

* Makes normal line parsing LHS sensitive

* Removes outdated note on block comments in a single line

* Wait a moment before executing each code lenses

* Sorting tests in order

* Sorts lenses in order

* Reverted to use executCmd

* Changes sorting logic

* Fixes test case: trailing space

* Dummy commit to re-invoke CI

* expect fail CPP Eval on Windows

* Corrects typo

* Test for #1258

* Corrects test header

* Ad-hoc treatment for ending brace in nested comment block

* `goldenTest` function from Eval plugin doesn't support multiple tests in the same block but in separate group

* Dummy commit to rerun CI

* Stop using CPP and use `knownBrokenForGhcVersions` and `knownBrokenOnWindows`

* Nested `expectedFailure` didn't work as expected

* Abolishes `Parser` type synonym

* Removes unneccesary comment evals

* Skip failed curentRange resolution

Co-authored-by: Junyoung/Clare Jang <[email protected]>

ghcide/src/Development/IDE/Core/Rules.hs

@@ -339,6 +339,9 @@ withOptHaddock = withOption Opt_Haddock
 withOption :: GeneralFlag -> ModSummary -> ModSummary
 withOption opt ms = ms{ms_hspp_opts= gopt_set (ms_hspp_opts ms) opt}
 
+withoutOption :: GeneralFlag -> ModSummary -> ModSummary
+withoutOption opt ms = ms{ms_hspp_opts= gopt_unset (ms_hspp_opts ms) opt}
+
 -- | Given some normal parse errors (first) and some from Haddock (second), merge them.
 --   Ignore Haddock errors that are in both. Demote Haddock-only errors to warnings.
 mergeParseErrorsHaddock :: [FileDiagnostic] -> [FileDiagnostic] -> [FileDiagnostic]
@@ -360,7 +363,7 @@ getParsedModuleWithCommentsRule = defineEarlyCutoff $ \GetParsedModuleWithCommen
     sess <- use_ GhcSession file
     opt <- getIdeOptions
 
-    let ms' = withOption Opt_KeepRawTokenStream ms
+    let ms' = withoutOption Opt_Haddock $ withOption Opt_KeepRawTokenStream ms
 
     liftIO $ getParsedModuleDefinition (hscEnv sess) opt file ms'
 

ghcide/src/Development/IDE/GHC/Orphans.hs

@@ -22,6 +22,7 @@ import qualified StringBuffer               as SB
 import Data.Text (Text)
 import Data.String (IsString(fromString))
 import Retrie.ExactPrint (Annotated)
+import Data.List (foldl')
 
 
 -- Orphan instances for types from the GHC API.

plugins/hls-eval-plugin/hls-eval-plugin.cabal

@@ -31,9 +31,7 @@ library
     Ide.Plugin.Eval.CodeLens
     Ide.Plugin.Eval.GHC
     Ide.Plugin.Eval.Parse.Option
-    Ide.Plugin.Eval.Parse.Parser
-    Ide.Plugin.Eval.Parse.Section
-    Ide.Plugin.Eval.Parse.Token
+    Ide.Plugin.Eval.Parse.Comments
     Ide.Plugin.Eval.Types
     Ide.Plugin.Eval.Util
 
@@ -44,6 +42,7 @@ library
     , deepseq
     , Diff
     , directory
+    , dlist
     , extra
     , filepath
     , ghc
@@ -54,6 +53,8 @@ library
     , haskell-lsp
     , haskell-lsp-types
     , hls-plugin-api
+    , lens
+    , megaparsec >= 0.9
     , parser-combinators
     , pretty-simple
     , QuickCheck
@@ -63,6 +64,7 @@ library
     , text
     , time
     , transformers
+    , mtl
     , unordered-containers
 
   ghc-options: -Wall -Wno-name-shadowing

plugins/hls-eval-plugin/src/Ide/Plugin/Eval/Code.hs

@@ -15,20 +15,21 @@ import GhcMonad (Ghc, GhcMonad, liftIO)
 import Ide.Plugin.Eval.Types (
     Language (Plain),
     Loc,
-    Located (Located),
     Section (sectionLanguage),
-    Test (Example, Property, testOutput),
+    Test (..),
     Txt,
     locate,
-    locate0,
+    locate0, Located(..)
  )
 import InteractiveEval (runDecls)
 import Unsafe.Coerce (unsafeCoerce)
+import Control.Lens ((^.))
+import Language.Haskell.LSP.Types.Lens (start, line)
 
 -- | Return the ranges of the expression and result parts of the given test
-testRanges :: Loc Test -> (Range, Range)
-testRanges (Located line tst) =
-    let startLine = line
+testRanges :: Test -> (Range, Range)
+testRanges tst =
+    let startLine = testRange tst ^. start.line
         (exprLines, resultLines) = testLenghts tst
         resLine = startLine + exprLines
      in ( Range
@@ -44,7 +45,7 @@ testRanges (Located line tst) =
 -}
 
 -- |The document range where the result of the test is defined
-resultRange :: Loc Test -> Range
+resultRange :: Test -> Range
 resultRange = snd . testRanges
 
 -- TODO: handle BLANKLINE
@@ -66,18 +67,18 @@ testCheck (section, test) out
     | otherwise = showDiffs $ getDiff (map T.pack $ testOutput test) out
 
 testLenghts :: Test -> (Int, Int)
-testLenghts (Example e r) = (NE.length e, length r)
-testLenghts (Property _ r) = (1, length r)
+testLenghts (Example e r _) = (NE.length e, length r)
+testLenghts (Property _ r _) = (1, length r)
 
 -- |A one-line Haskell statement
 type Statement = Loc String
 
-asStatements :: Loc Test -> [Statement]
-asStatements lt = locate (asStmts <$> lt)
+asStatements :: Test -> [Statement]
+asStatements lt = locate $ Located (testRange lt ^. start.line) (asStmts lt)
 
 asStmts :: Test -> [Txt]
-asStmts (Example e _) = NE.toList e
-asStmts (Property t _) =
+asStmts (Example e _ _) = NE.toList e
+asStmts (Property t _ _) =
     ["prop11 = " ++ t, "(propEvaluation prop11 :: IO String)"]
 
 -- |Evaluate an expression (either a pure expression or an IO a)