fixed: additional parsing cases, module reference parsing

added: tests for errors, which were highlighted during parsing

Resolves: #355

src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeAPI.kt

@@ -138,13 +138,7 @@ object SnakemakeAPI {
     /**
      * For uses parsing
      */
-    val USE_SECTIONS_KEYWORDS = RULE_OR_CHECKPOINT_SECTION_KEYWORDS - setOf(
-        SECTION_SHELL,
-        SECTION_NOTEBOOK,
-        SECTION_SCRIPT,
-        SECTION_CWL,
-        SECTION_RUN
-    )
+    val USE_SECTIONS_KEYWORDS = RULE_OR_CHECKPOINT_SECTION_KEYWORDS - EXECUTION_SECTIONS_KEYWORDS - SECTION_RUN
 
     /**
      * For type inference:

src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkStatementParsing.kt

@@ -163,9 +163,9 @@ class SmkStatementParsing(
         nextToken()
 
         // Parse second word in 'use rule'
-        if (section.sectionKeyword == SmkTokenTypes.USE_KEYWORD) {
+        if (section == useSectionParsingData) {
             if (myBuilder.tokenText != SnakemakeNames.RULE_KEYWORD) {
-                myBuilder.error("Unexpected token '${myBuilder.tokenText}' after 'use' keyword")
+                myBuilder.error(SnakemakeBundle.message("PARSE.use.rule.keyword.expected"))
             } else {
                 myBuilder.remapCurrentToken(SmkTokenTypes.RULE_KEYWORD)
                 nextToken()
@@ -310,17 +310,26 @@ class SmkStatementParsing(
 //    }
 
 
-    private fun parseIdentifier(type: IElementType = SmkElementTypes.REFERENCE_EXPRESSION): Boolean {
+    private fun parseIdentifier(): Boolean {
         val referenceMarker = myBuilder.mark()
         if (Parsing.isIdentifier(myBuilder)) {
             Parsing.advanceIdentifierLike(myBuilder)
-            referenceMarker.done(type)
+            referenceMarker.done(SmkElementTypes.REFERENCE_EXPRESSION)
             return true
         }
         referenceMarker.drop()
         return false
     }
 
+    private fun checkMatchesAndRemapToken(required: PyElementType, remapTo: PyElementType, nameForUser: String) {
+        if (myBuilder.tokenType == required) {
+            myBuilder.remapCurrentToken(remapTo)
+        } else {
+            myBuilder.error(PyPsiBundle.message("PARSE.0.expected", nameForUser))
+        }
+        nextToken()
+    }
+
     /**
      * Parsing 'use' statement. Starts after first identifier and ends before the colon
      */
@@ -336,10 +345,10 @@ class SmkStatementParsing(
         }
 
         if (myBuilder.tokenType != PyTokenTypes.AS_KEYWORD) {
-            checkMatches(PyTokenTypes.FROM_KEYWORD, PyPsiBundle.message("PARSE.0.expected", "from"))
+            checkMatchesAndRemapToken(PyTokenTypes.FROM_KEYWORD, SmkTokenTypes.SMK_FROM_KEYWORD, "from")
 
             // Creates reference to module definition
-            if (!parseIdentifier(PyElementTypes.REFERENCE_EXPRESSION)) {
+            if (!parseIdentifier()) {
                 myBuilder.error(PyPsiBundle.message("PARSE.expected.identifier"))
             }
 
@@ -348,36 +357,69 @@ class SmkStatementParsing(
         // If there are no 'module' keyword, we expect 'as' keyword
         // If there are 'module' keyword, there may not be 'as' keyword
         if (!hasImport || myBuilder.tokenType == PyTokenTypes.AS_KEYWORD) {
-            checkMatches(PyTokenTypes.AS_KEYWORD, PyPsiBundle.message("PARSE.0.expected", "as"))
+            checkMatchesAndRemapToken(PyTokenTypes.AS_KEYWORD, SmkTokenTypes.SMK_AS_KEYWORD, "as")
 
-            // New rule name can be: text_*, *_text, text_*_text, text or *
-            val hasFirstIdentifier = myBuilder.tokenType == PyTokenTypes.IDENTIFIER
+            // New rule name can be: text, *, *_text_* and so on
+            var lasTokenIsIdentifier =
+                myBuilder.tokenType != PyTokenTypes.IDENTIFIER // Default value need to ve reversed
+            var simpleName = true // Does new rule name consist of one identifier
+            var hasIdentifier = false // Do we have new rule name
             val name = myBuilder.mark()
-            if (hasFirstIdentifier) {
-                nextToken()
-            }
-            if (myBuilder.tokenType == PyTokenTypes.MULT) {
-                nextToken()
-                if (myBuilder.tokenType == PyTokenTypes.IDENTIFIER) {
-                    nextToken()
+            while (true) {
+                when (myBuilder.tokenType) {
+                    PyTokenTypes.IDENTIFIER -> {
+                        if (lasTokenIsIdentifier) {
+                            break // Because it's separated by whitespace so it isn't name anymore
+                        }
+                        lasTokenIsIdentifier = true
+                        hasIdentifier = true
+                        nextToken()
+                    }
+                    PyTokenTypes.MULT -> {
+                        if (!lasTokenIsIdentifier) {
+                            myBuilder.error(SnakemakeBundle.message("PARSE.use.double.mult.sign"))
+                        }
+                        lasTokenIsIdentifier = false
+                        hasIdentifier = true
+                        simpleName = false
+                        nextToken()
+                    }
+                    PyTokenTypes.EXP -> {
+                        myBuilder.error(SnakemakeBundle.message("PARSE.use.double.mult.sign"))
+                        lasTokenIsIdentifier = false
+                        simpleName = false
+                        nextToken()
+                    }
+                    else -> break
                 }
-                name.done(SmkElementTypes.USE_NAME_IDENTIFIER)
-            } else {
+            }
+            if (!hasIdentifier) { // No identifiers and/or '*' symbols
                 name.drop()
-                if (!hasFirstIdentifier) {
-                    checkMatches(PyTokenTypes.IDENTIFIER, PyPsiBundle.message("PARSE.expected.identifier"))
+                myBuilder.error(PyPsiBundle.message("PARSE.expected.identifier"))
+            } else {
+                if (!simpleName) { // New rule name contains at least one '*' symbol
+                    name.done(SmkElementTypes.USE_NAME_IDENTIFIER)
+                } else { // New rule name consists of one identifier
+                    name.drop()
                 }
             }
         }
 
-        if (myBuilder.tokenType == PyTokenTypes.WITH_KEYWORD) {
-            if (listOfRules) {
-                myBuilder.error(SnakemakeBundle.message("PARSE.use.with.not.allowed"))
+        return when (myBuilder.tokenType) {
+            PyTokenTypes.WITH_KEYWORD -> {
+                myBuilder.remapCurrentToken(SmkTokenTypes.SMK_WITH_KEYWORD)
+                if (listOfRules) {
+                    myBuilder.error(SnakemakeBundle.message("PARSE.use.with.not.allowed"))
+                }
+                nextToken()
+                true
             }
-            nextToken()
-            return true
+            PyTokenTypes.COLON -> {
+                myBuilder.error(PyPsiBundle.message("PARSE.0.expected", "with"))
+                true
+            }
+            else -> false
         }
-        return false
     }
 }
 

src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkTokenTypes.kt

@@ -15,6 +15,10 @@ object SmkTokenTypes {
     val MODULE_KEYWORD = PyElementType("MODULE_KEYWORD")
     val USE_KEYWORD = PyElementType("USE_KEYWORD")
 
+    val SMK_FROM_KEYWORD = PyElementType("SMK_FROM_KEYWORD")
+    val SMK_AS_KEYWORD = PyElementType("SMK_AS_KEYWORD")
+    val SMK_WITH_KEYWORD = PyElementType("SMK_WITH_KEYWORD")
+
     val WORKFLOW_CONFIGFILE_KEYWORD = PyElementType("WORKFLOW_CONFIGFILE_KEYWORD")
     val WORKFLOW_REPORT_KEYWORD = PyElementType("WORKFLOW_REPORT_KEYWORD")
     val WORKFLOW_WILDCARD_CONSTRAINTS_KEYWORD = PyElementType("WORKFLOW_WILDCARD_CONSTRAINTS_KEYWORD")

src/main/kotlin/com/jetbrains/snakecharm/lang/psi/impl/refs/SmkRuleOrCheckpointNameReference.kt

@@ -4,7 +4,6 @@ import com.intellij.lang.annotation.HighlightSeverity
 import com.intellij.psi.util.elementType
 import com.jetbrains.python.PyElementTypes
 import com.jetbrains.python.psi.AccessDirection
-import com.jetbrains.python.psi.PyReferenceExpression
 import com.jetbrains.python.psi.impl.references.PyReferenceImpl
 import com.jetbrains.python.psi.resolve.PyResolveContext
 import com.jetbrains.python.psi.resolve.RatedResolveResult
@@ -50,11 +49,25 @@ class SmkRuleOrCheckpointNameReference(
         results.addAll(SmkRulesType(null, smkFile).resolveMember(name, element, ctx, myContext))
         results.addAll(SmkCheckpointType(null, smkFile).resolveMember(name, element, ctx, myContext))
         results.addAll(SmkUsesType(null, smkFile).resolveMember(name, element, ctx, myContext))
+        results.addAll(collectModulesAndResolveThem(smkFile, name))
         results.addAll(collectModuleFromUseSection(element))
 
         return results
     }
 
+    /**
+     * Collects all modules sections names from local file which name is [name]
+     */
+    private fun collectModulesAndResolveThem(smkFile: SmkFile, name: String): List<RatedResolveResult> {
+        val modules = smkFile.collectModules().map { it.second }.filter { elem -> elem.name == name }
+        if (modules.isEmpty()) {
+            return emptyList()
+        }
+        return modules.map { element ->
+            RatedResolveResult(SmkResolveUtil.RATE_NORMAL, element)
+        }
+    }
+
     /**
      * Resolve rule reference, which is declared in 'use' section.
      * It refers to module, which imports such rule.
@@ -69,12 +82,7 @@ class SmkRuleOrCheckpointNameReference(
                 moduleRef = moduleRef.nextSibling
             }
             if (moduleRef != null) {
-                return listOf(
-                    RatedResolveResult(
-                        SmkResolveUtil.RATE_NORMAL,
-                        PyReferenceImpl(moduleRef as PyReferenceExpression, myContext).resolve()
-                    )
-                )
+                return SmkRuleOrCheckpointNameReference(moduleRef as SmkReferenceExpression, myContext).resolveInner()
             }
         }
 

src/main/resources/SnakemakeBundle.properties

@@ -182,6 +182,8 @@ PARSE.incorrect.unindent=Unindent does not match any outer indentation level.
 PARSE.eof.docstring=Docstring at end of file does not precede any statement
 PARSE.rule.expected.rule.commend.to.docstring=Expecting rule keyword, comment or docstrings inside a rule definition.
 PARSE.use.with.not.allowed=Keyword 'with' is not allowed with rule pattern '*'
+PARSE.use.double.mult.sign=Can't be '*' after '*' in the new rule name
+PARSE.use.rule.keyword.expected='rule' keyword is expected after 'use'
 
 # String language parsing
 SMKSL.PARSE.expected.identifier.name=Expected identifier name

src/test/resources/features/highlighting/inspections/unrecognized_section_inspection.feature

@@ -70,6 +70,7 @@ Feature: Inspection if section isn't recognized by SnakeCharm
           notebook: ""
           script: ""
           cwl: ""
+          wrapper: ""
       """
       And SmkUnrecognizedSectionInspection inspection is enabled
       Then I expect inspection weak warning on <run> with message
@@ -92,4 +93,8 @@ Feature: Inspection if section isn't recognized by SnakeCharm
       """
       Section 'cwl' isn't recognized by SnakeCharm plugin or there could be a typo in the section name.
       """
+      Then I expect inspection weak warning on <wrapper> with message
+      """
+      Section 'wrapper' isn't recognized by SnakeCharm plugin or there could be a typo in the section name.
+      """
       When I check highlighting weak warnings

src/test/resources/features/highlighting/smk_parsing_error.feature

@@ -0,0 +1,66 @@
+Feature: Checks syntax errors, which were detected during parsing
+
+  Scenario: Check 'use' section errors highlighting
+    Given a snakemake project
+    Given I open a file "foo.smk" with text
+    """
+    module MODULE:
+        snakefile: "file.smk"
+
+    use a1 as NAME
+
+    use rule as NAME2
+
+    use rule NAME frm MODULE as NAME3
+
+    use rule NAME from # Module name
+
+    use rule * from MODULE with:
+        input: "datafile.doc"
+
+    use rule NAME as # No identifier
+
+    use rule NAME as d**
+    """
+    Then I expect inspection error on < > in < a1> with message
+    """
+    'rule' keyword is expected after 'use'
+    """
+    Then I expect inspection error on < > in < as NAME2> with message
+    """
+    '*' or rule name expected
+    """
+    Then I expect inspection error on < > in < frm> with message
+    """
+    'from' expected
+    """
+    Then I expect inspection error on < > in <from # Module name> with message
+    """
+    Identifier expected
+    """
+    Then I expect inspection error on < > in <MODULE with:> with message
+    """
+    Keyword 'with' is not allowed with rule pattern '*'
+    """
+    Then I expect inspection error on < > in <as # No identifier> with message
+    """
+    Identifier expected
+    """
+    Then I expect inspection error on <*> in <d**> with message
+    """
+    Can't be '*' after '*' in the new rule name
+    """
+    When I check highlighting errors
+
+  Scenario: Check 'use' section errors highlighting part 2. Missed 'with' keyword
+    Given a snakemake project
+    Given I open a file "foo.smk" with text
+    """
+    use rule NAME as NAME4:
+        input: "data_file3.doc"
+    """
+    Then I expect inspection error with message "'with' expected" on
+    """
+    :
+    """
+    When I check highlighting errors

src/test/resources/features/resolve/use_name_resolve.feature → src/test/resources/features/resolve/uses_and_modules_name_resolve.feature

@@ -1,4 +1,4 @@
-Feature: Resolve use name to its declaration
+Feature: Resolve use and module name to its declaration
   Scenario: Refer to rule section
     Given a snakemake project
     Given I open a file "foo.smk" with text
@@ -42,4 +42,21 @@ Feature: Resolve use name to its declaration
         "data_file.txt"
     """
     When I put the caret at NAME
+    Then reference should resolve to "MODULE:" in "foo.smk"
+
+  Scenario: Module name refere to module declaration
+    Given a snakemake project
+    Given I open a file "foo.smk" with text
+    """
+    module MODULE:
+      snakefile:
+        "../path/to/otherworkflow/Snakefile"
+      configfile:
+        "path/to/custom_configfile.yaml"
+
+    use rule NAME from MODULE as other with:
+      input:
+        "data_file.txt"
+    """
+    When I put the caret at MODULE as
     Then reference should resolve to "MODULE:" in "foo.smk"