From d07b7430b29090f251391e4fc9bb49431ca9ab2f Mon Sep 17 00:00:00 2001 From: Roman Chernyatchik Date: Mon, 21 Oct 2024 21:25:45 +0200 Subject: [PATCH] refact: 1) use YAML description for top-level sections list. Also warn user that feature request should be subitted. 2) add missing configfile annotation --- snakemake_api.yaml | 6 + .../snakecharm/actions/SmkStatementMover.kt | 3 +- .../snakecharm/codeInsight/SnakemakeApi.kt | 122 +++++++++++------- .../SmkKeywordCompletionContributor.kt | 9 +- .../SmkAndSmkSLFindUsagesProvider.kt | 4 +- .../SnakemakeApiYamlAnnotationsService.kt | 13 ++ .../lang/parser/SmkParserContext.kt | 4 +- .../lang/parser/SmkParserDefinition.kt | 2 +- .../lang/parser/SmkStatementParsing.kt | 24 +++- .../snakecharm/lang/parser/SnakemakeLexer.kt | 2 +- .../snakecharm/lang/parser/SnakemakeParser.kt | 5 +- .../lang/parser/SnakemakeLexerTest.kt | 19 +++ 12 files changed, 147 insertions(+), 66 deletions(-) diff --git a/snakemake_api.yaml b/snakemake_api.yaml index c2d70ea82..04b653cd7 100644 --- a/snakemake_api.yaml +++ b/snakemake_api.yaml @@ -473,6 +473,12 @@ changelog: - name: "workdir" type: "top-level" multiple_args_allowed: False + keyword_args_allowed: False + + - name: "configfile" + type: "top-level" + multiple_args_allowed: False + keyword_args_allowed: False - name: "subworkflow" type: "top-level" diff --git a/src/main/kotlin/com/jetbrains/snakecharm/actions/SmkStatementMover.kt b/src/main/kotlin/com/jetbrains/snakecharm/actions/SmkStatementMover.kt index bf8c36e17..7b87f8ccc 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/actions/SmkStatementMover.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/actions/SmkStatementMover.kt @@ -15,7 +15,6 @@ import com.intellij.psi.util.PsiTreeUtil import com.jetbrains.python.codeInsight.editorActions.moveUpDown.PyStatementMover import com.jetbrains.python.psi.* import com.jetbrains.snakecharm.codeInsight.SnakemakeApi -import com.jetbrains.snakecharm.codeInsight.SnakemakeApi.TOPLEVEL_ARGS_SECTION_KEYWORDS import com.jetbrains.snakecharm.codeInsight.SnakemakeApiService import com.jetbrains.snakecharm.lang.psi.* @@ -194,7 +193,7 @@ open class SmkStatementMover : PyStatementMover() { // do not move sections that cannot be toplevel: if (elementToMove is SmkArgsSection) { val keyword = elementToMove.sectionKeyword - val sectionCouldBeToplevel = keyword != null && keyword in TOPLEVEL_ARGS_SECTION_KEYWORDS + val sectionCouldBeToplevel = keyword != null && keyword in api.getAllPossibleToplevelArgsSectionKeywords() if (!sectionCouldBeToplevel) { return false } diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeApi.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeApi.kt index 098d0ae60..f9bd5b301 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeApi.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/SnakemakeApi.kt @@ -1,6 +1,7 @@ package com.jetbrains.snakecharm.codeInsight import com.intellij.openapi.Disposable +import com.intellij.openapi.application.ApplicationManager import com.intellij.openapi.components.Service import com.intellij.openapi.project.Project import com.intellij.openapi.util.Disposer @@ -10,6 +11,7 @@ import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings import com.jetbrains.snakecharm.framework.SmkSupportProjectSettingsListener import com.jetbrains.snakecharm.framework.SnakemakeApiYamlAnnotationsService import com.jetbrains.snakecharm.framework.snakemakeAPIAnnotations.SmkApiAnnotationKeywordDeprecationParams +import com.jetbrains.snakecharm.framework.snakemakeAPIAnnotations.SmkApiAnnotationKeywordIntroductionParams import com.jetbrains.snakecharm.framework.snakemakeAPIAnnotations.SmkApiAnnotationParsingContextType import com.jetbrains.snakecharm.framework.snakemakeAPIAnnotations.SmkApiAnnotationParsingContextType.TOP_LEVEL import com.jetbrains.snakecharm.lang.SmkLanguageVersion @@ -39,7 +41,6 @@ import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_SINGULARITY_KEYWORD import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_WILDCARD_CONSTRAINTS_KEYWORD import com.jetbrains.snakecharm.lang.SnakemakeNames.WORKFLOW_WORKDIR_KEYWORD import com.jetbrains.snakecharm.lang.parser.SnakemakeLexer -import io.ktor.util.* import kotlinx.collections.immutable.toImmutableMap import kotlinx.collections.immutable.toImmutableSet @@ -77,9 +78,20 @@ object SnakemakeApi { "snakemake.io.Resources" to "resources" ) - // List of top-level sections - // XXX: cannot move to SnakemakeApiService because it is used to create Lexer, Parser, WordScanner, Highlighter - // and not everywhere we could pass project instance + // List of top-level sections. + // XXX: cannot fully move to SnakemakeApiService because it is used to create Lexer, WordScanner, Highlighter + // and not everywhere we could pass project instance. + // + // We could pass YAML settings to: + // * Parser, but it is capable of parsing unrecognized sections as well (just slightly slower). + // Cannot parse project: + // * WordScanner: Not a problem, lexer reprots them as Py:IDENTIFIER and wordscanner accepts it + // * Lexer: has better support for hardcoded sections, but also covers unrecognized pretty well + // * Highlighter: will not highlight unregognized things. We use annotator to highlight missing things based on parser advanced info + // + // So better to add new top-level sections here, but we could first register in YAML file and get exception for + // unregognized thigs. Such exceptions could be report to EA by user & we could update internal map. + val TOPLEVEL_ARGS_SECTION_KEYWORDS = setOf( WORKFLOW_CONFIGFILE_KEYWORD, WORKFLOW_REPORT_KEYWORD, @@ -185,21 +197,11 @@ class SnakemakeApiService(val project: Project): Disposable { } @Suppress("unused") - fun isTopLevelArgsSectionKeyword(keyword: String): Boolean { - // XXX: Used in parsing/lexing, cannot collect from YAML file - return keyword in SnakemakeApi.TOPLEVEL_ARGS_SECTION_KEYWORDS - } + fun getTopLevelsKeywords(): Set = state.contextType2SectionKeywords[TOP_LEVEL.typeStr] ?: emptySet() - fun isTopLevelKeyword(keyword: String): Boolean { - // XXX: Used in parsing/lexing, cannot collect from YAML file - return keyword in SnakemakeLexer.KEYWORD_LIKE_SECTION_NAME_2_TOKEN_TYPE - } + fun getTopLevelArgsSectionsKeywords(): Set = state.contextType2ArgsSectionKeywords[TOP_LEVEL.typeStr] ?: emptySet() - @Suppress("unused") - fun getTopLevelsKeywords(): Set { - // XXX: Used in parsing/lexing, cannot collect from YAML file - return SnakemakeLexer.KEYWORD_LIKE_SECTION_NAME_2_TOKEN_TYPE.keys.unmodifiable() - } + fun getAllPossibleToplevelArgsSectionKeywords(): Set = state.allPossibleToplevelArgsSectionKeywords fun getSubsectionPossibleLambdaParamNames(): Set = state.subsectionsAllPossibleArgNames @@ -318,34 +320,37 @@ class SnakemakeApiService(val project: Project): Disposable { fun getAllPossibleModuleSectionKeywords(): Set = state.allPossibleModuleSectionKeywords @Suppress("unused") - fun getModuleAllSectionTypesKeywords(): Set = state.contextType2SubsectionKeywords[MODULE_KEYWORD] ?: emptySet() + fun getModuleAllSectionTypesKeywords(): Set = state.contextType2SectionKeywords[MODULE_KEYWORD] ?: emptySet() @Suppress("unused") - fun getModuleArgsSectionKeywords(): Set = state.contextType2ArgsSubsectionKeywords[MODULE_KEYWORD] ?: emptySet() + fun getModuleArgsSectionKeywords(): Set = state.contextType2ArgsSectionKeywords[MODULE_KEYWORD] ?: emptySet() fun getAllPossibleRuleOrCheckpointArgsSectionKeywords(): Set = state.allPossibleRuleOrCheckpointSectionKeywords @Suppress("unused") fun getRuleOrCheckpointAllSectionTypesKeywords(): Set = - (state.contextType2SubsectionKeywords[RULE_KEYWORD] ?: emptySet()) + - (state.contextType2SubsectionKeywords[CHECKPOINT_KEYWORD] ?: emptySet()) + (state.contextType2SectionKeywords[RULE_KEYWORD] ?: emptySet()) + + (state.contextType2SectionKeywords[CHECKPOINT_KEYWORD] ?: emptySet()) fun getRuleOrCheckpointArgsSectionKeywords(): Set = - (state.contextType2ArgsSubsectionKeywords[RULE_KEYWORD] ?: emptySet()) + - (state.contextType2ArgsSubsectionKeywords[CHECKPOINT_KEYWORD] ?: emptySet()) + (state.contextType2ArgsSectionKeywords[RULE_KEYWORD] ?: emptySet()) + + (state.contextType2ArgsSectionKeywords[CHECKPOINT_KEYWORD] ?: emptySet()) fun getAllPossibleUseSectionKeywordsIncludingExecSections() = state.allPossibleUseSectionKeywordsIncludingExecSections - fun getUseSectionKeywords(): Set = state.contextType2SubsectionKeywords[USE_KEYWORD] ?: emptySet() + fun getUseSectionKeywords(): Set = state.contextType2SectionKeywords[USE_KEYWORD] ?: emptySet() private fun doRefresh(version: String?) { + val toplevelIntroductions: List>>? + val newState = if (version == null) { + toplevelIntroductions = null SnakemakeApiStateForLangLevel.EMPTY } else { val yamlApi = SnakemakeApiYamlAnnotationsService.getInstance() - val contextType2SubsectionKeywords = HashMap>() - val contextType2ArgsSubsectionKeywords = HashMap>() + val contextType2SectionKeywords = HashMap>() + val contextType2ArgsSectionKeywords = HashMap>() val contextType2SingleArgSectionKeywords = HashMap>() val contextType2PositionalOnlySectionKeywords = HashMap>() val contextTypeAndSubsection2LambdaArgs = HashMap>() @@ -360,7 +365,7 @@ class SnakemakeApiService(val project: Project): Disposable { val smkLangVers = SmkLanguageVersion(version) // add top-level data: - val toplevelIntroductions = yamlApi.getToplevelIntroductions(smkLangVers) + toplevelIntroductions = yamlApi.getToplevelIntroductions(smkLangVers) contextType2SingleArgSectionKeywords.put( TOP_LEVEL.typeStr, toplevelIntroductions.mapNotNull { (directiveKeyword, versAndParams) -> @@ -376,14 +381,13 @@ class SnakemakeApiService(val project: Project): Disposable { }.toMutableList() ) - contextType2SubsectionKeywords.put( + contextType2SectionKeywords.put( TOP_LEVEL.typeStr, - toplevelIntroductions.map { (directiveKeyword, _) -> + (toplevelIntroductions.map { (directiveKeyword, _) -> directiveKeyword - }.toMutableSet() + } + SnakemakeLexer.SPECIAL_KEYWORDS_2_TOKEN_TYPE.keys).toMutableSet() ) - - contextType2ArgsSubsectionKeywords.put( + contextType2ArgsSectionKeywords.put( TOP_LEVEL.typeStr, toplevelIntroductions.mapNotNull { (directiveKeyword, versAndParams) -> val (_, params) = versAndParams @@ -391,17 +395,6 @@ class SnakemakeApiService(val project: Project): Disposable { }.toMutableSet() ) - toplevelIntroductions.forEach { (directiveKeyword, _) -> - require(isTopLevelKeyword(directiveKeyword)) { - "YAML format error: '$directiveKeyword' should be one of" + - " [${ - SnakemakeLexer.KEYWORD_LIKE_SECTION_NAME_2_TOKEN_TYPE.keys.sorted() - .joinToString(separator = ", ") - }]. " + - "Please file a feature request at https://github.com/JetBrains-Research/snakecharm/issues" - } - } - // add subsections data: val subsectionIntroductions = yamlApi.getSubsectionsIntroductions(smkLangVers) subsectionIntroductions.forEach { (ctxAndName, versAndParams) -> @@ -411,11 +404,11 @@ class SnakemakeApiService(val project: Project): Disposable { if (context != USE_KEYWORD || !params.isExecutionSection) { // for USE block - do not add execution sections, they are not supported // for rules/checkpoints - add all - contextType2SubsectionKeywords.getOrPut(context) { mutableSetOf() }.add(directiveKeyword) + contextType2SectionKeywords.getOrPut(context) { mutableSetOf() }.add(directiveKeyword) } if (params.isArgsSection) { - val keywords = contextType2ArgsSubsectionKeywords.getOrPut(context) { mutableSetOf() } + val keywords = contextType2ArgsSectionKeywords.getOrPut(context) { mutableSetOf() } keywords.add(directiveKeyword) } @@ -502,10 +495,11 @@ class SnakemakeApiService(val project: Project): Disposable { val allPossibleModuleSectionKeywords = yamlApi.collectAllPossibleModuleSubsectionKeywords() + val allPossibleToplevelArgsSectionKeywords = yamlApi.collectAllPossibleTopLevelArgsSectionsKeywords() + SnakemakeApi.TOPLEVEL_ARGS_SECTION_KEYWORDS SnakemakeApiStateForLangLevel( - contextType2SubsectionKeywords = contextType2SubsectionKeywords.toImmutableMap(), - contextType2ArgsSubsectionKeywords = contextType2ArgsSubsectionKeywords.toImmutableMap(), + contextType2SectionKeywords = contextType2SectionKeywords.toImmutableMap(), + contextType2ArgsSectionKeywords = contextType2ArgsSectionKeywords.toImmutableMap(), contextType2SingleArgSectionKeywords = contextType2SingleArgSectionKeywords.toImmutableMap(), contextType2PositionalOnlySectionKeywords = contextType2PositionalOnlySectionKeywords.toImmutableMap(), contextTypeAndSubsection2LambdaArgs = contextTypeAndSubsection2LambdaArgs.toImmutableMap(), @@ -524,9 +518,36 @@ class SnakemakeApiService(val project: Project): Disposable { allPossibleUseSectionKeywordsIncludingExecSections = allPossibleUseSectionKeywordsIncludingExecSections.toImmutableSet(), allPossibleModuleSectionKeywords = allPossibleModuleSectionKeywords.toImmutableSet(), allPossibleExecutionSectionKeywords = allPossibleExecutionSectionKeywords.toImmutableSet(), + allPossibleToplevelArgsSectionKeywords = allPossibleToplevelArgsSectionKeywords.toImmutableSet(), ) } state = newState + + if (toplevelIntroductions != null) { + val unregognizedTopLevelSections = mutableSetOf() + // collect unregonized things + toplevelIntroductions.forEach { (directiveKeyword, _) -> + if (directiveKeyword !in SnakemakeLexer.KEYWORD_LIKE_SECTION_NAME_2_TOKEN_TYPE) { + unregognizedTopLevelSections.add(directiveKeyword) + } + } + + if (unregognizedTopLevelSections.isNotEmpty()) { + // XXX: report error, so user could submit it via EA + ApplicationManager.getApplication().invokeLater() { + error( + "YAML format error: Top-Level directives '${ + unregognizedTopLevelSections.sorted().joinToString() + }' missing in recognized directives list: " + + " [${ + SnakemakeLexer.KEYWORD_LIKE_SECTION_NAME_2_TOKEN_TYPE.keys.sorted() + .joinToString(separator = ", ") + }]. " + + "Please file a feature request at https://github.com/JetBrains-Research/snakecharm/issues" + ) + } + } + } } fun initOnStartup(smkSettings: SmkSupportProjectSettings) { @@ -580,8 +601,8 @@ internal data class SnakemakeApiStateForLangLevel( val contextType2ExecutionSectionSubsectionKeywords: Map>, val contextType2AccessibleInRuleObjectSubsectionKeywords: Map>, val contextType2AccessibleAccessibleAsPlaceholderSubsectionKeywords: Map>, - val contextType2SubsectionKeywords: Map>, - val contextType2ArgsSubsectionKeywords: Map>, + val contextType2SectionKeywords: Map>, + val contextType2ArgsSectionKeywords: Map>, val funFqnToSectionRestrictionList: Map>, val funShortNameDeprecations: Map>, val funFqnDeprecations: Map>, @@ -591,6 +612,7 @@ internal data class SnakemakeApiStateForLangLevel( val allPossibleUseSectionKeywordsIncludingExecSections: Set, val allPossibleModuleSectionKeywords: Set, val allPossibleExecutionSectionKeywords: Set, + val allPossibleToplevelArgsSectionKeywords: Set, ) { val subsectionsAllPossibleArgNames = contextTypeAndSubsection2LambdaArgs.values.flatMap { it.asIterable() }.toImmutableSet() @@ -598,7 +620,7 @@ internal data class SnakemakeApiStateForLangLevel( val EMPTY = SnakemakeApiStateForLangLevel( emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptySet(), emptySet(), emptySet(), - emptySet(), emptySet(), emptySet() + emptySet(), emptySet(), emptySet(), emptySet() ) } } \ No newline at end of file diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt index eda3ad1be..6d86d2f11 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/completion/SmkKeywordCompletionContributor.kt @@ -19,7 +19,6 @@ import com.jetbrains.python.PyTokenTypes import com.jetbrains.python.codeInsight.completion.PythonLookupElement import com.jetbrains.python.psi.* import com.jetbrains.snakecharm.codeInsight.SnakemakeApi.SUBWORKFLOW_SECTIONS_KEYWORDS -import com.jetbrains.snakecharm.codeInsight.SnakemakeApi.TOPLEVEL_ARGS_SECTION_KEYWORDS import com.jetbrains.snakecharm.codeInsight.SnakemakeApi.USE_DECLARATION_KEYWORDS import com.jetbrains.snakecharm.codeInsight.SnakemakeApiService import com.jetbrains.snakecharm.framework.SmkSupportProjectSettings @@ -113,16 +112,20 @@ object WorkflowTopLevelKeywordsProvider : CompletionProvider tokenType2Name[tt] - } + TOPLEVEL_ARGS_SECTION_KEYWORDS + } + api.getAllPossibleToplevelArgsSectionKeywords() val spaceTailKeys = RULE_LIKE.types.map { tt -> tokenType2Name[tt]!! } // top-level - val project = parameters.position.project listOf( colonAndWhiteSpaceTailKeys to ColonAndWhiteSpaceTail, spaceTailKeys to TailTypes.spaceType(), diff --git a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/refactoring/SmkAndSmkSLFindUsagesProvider.kt b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/refactoring/SmkAndSmkSLFindUsagesProvider.kt index f19cbc92d..2d0e5f158 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/refactoring/SmkAndSmkSLFindUsagesProvider.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/codeInsight/refactoring/SmkAndSmkSLFindUsagesProvider.kt @@ -46,8 +46,8 @@ class SmkAndSmkSLFindUsagesProvider : PythonFindUsagesProvider() { class SmkWordsScanner : DefaultWordsScanner( SnakemakeLexer(), TokenSet.orSet( - WORKFLOW_TOPLEVEL_DECORATORS, - TokenSet.create(PyTokenTypes.IDENTIFIER) + WORKFLOW_TOPLEVEL_DECORATORS, // e.g. hardcoded sections + TokenSet.create(PyTokenTypes.IDENTIFIER) // e.g. unrecognized top-level sections, etc. ), TokenSet.create(PyTokenTypes.END_OF_LINE_COMMENT), PyTokenTypes.STRING_NODES diff --git a/src/main/kotlin/com/jetbrains/snakecharm/framework/SnakemakeApiYamlAnnotationsService.kt b/src/main/kotlin/com/jetbrains/snakecharm/framework/SnakemakeApiYamlAnnotationsService.kt index be0f0e740..d268e3338 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/framework/SnakemakeApiYamlAnnotationsService.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/framework/SnakemakeApiYamlAnnotationsService.kt @@ -285,6 +285,19 @@ class SnakemakeApiYamlAnnotationsService( latestDeprecated?.key } + @Suppress("unused") + fun collectAllPossibleTopLevelSectionsKeywords(): Set = topLevelName2Introduction.keys + + fun collectAllPossibleTopLevelArgsSectionsKeywords(): Set { + val keywords = mutableSetOf() + topLevelName2Introduction.forEach { keyword, tree -> + if (tree.values.any() {it.isArgsSection}) { + keywords.add(keyword) + } + } + return keywords + } + fun collectAllPossibleUseSubsectionKeywordsIncludingExecutionSections(): Set = collectAllPossibleSubsectionKeywords { type -> type == USE_KEYWORD } diff --git a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkParserContext.kt b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkParserContext.kt index 96ffb08e1..1ab2a4070 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkParserContext.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkParserContext.kt @@ -2,6 +2,7 @@ package com.jetbrains.snakecharm.lang.parser import com.intellij.lang.SyntaxTreeBuilder import com.intellij.lang.impl.PsiBuilderImpl +import com.intellij.openapi.project.Project import com.jetbrains.python.parsing.ParsingContext import com.jetbrains.python.psi.LanguageLevel @@ -11,7 +12,8 @@ import com.jetbrains.python.psi.LanguageLevel */ class SmkParserContext( builder: SyntaxTreeBuilder, - languageLevel: LanguageLevel + languageLevel: LanguageLevel, + val project: Project ): ParsingContext(builder, languageLevel) { init { diff --git a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkParserDefinition.kt b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkParserDefinition.kt index c607bd426..21fcad010 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkParserDefinition.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkParserDefinition.kt @@ -14,7 +14,7 @@ import com.jetbrains.snakecharm.lang.psi.SmkFile class SmkParserDefinition: PythonParserDefinition() { override fun createLexer(project: Project) = SnakemakeLexer() - override fun createParser(project: Project): PsiParser = SnakemakeParser() + override fun createParser(project: Project): PsiParser = SnakemakeParser(project) override fun getFileNodeType() = SnakemakeLanguageDialect.fileElementType diff --git a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkStatementParsing.kt b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkStatementParsing.kt index 43be6a871..d81f649a0 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkStatementParsing.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SmkStatementParsing.kt @@ -7,8 +7,10 @@ import com.jetbrains.python.PyTokenTypes import com.jetbrains.python.parsing.StatementParsing import com.jetbrains.python.psi.PyElementType import com.jetbrains.snakecharm.SnakemakeBundle +import com.jetbrains.snakecharm.codeInsight.SnakemakeApiService import com.jetbrains.snakecharm.lang.SnakemakeNames import com.jetbrains.snakecharm.lang.parser.SmkTokenTypes.RULE_OR_CHECKPOINT +import com.jetbrains.snakecharm.lang.parser.SmkTokenTypes.WORKFLOW_TOPLEVEL_ARGS_SECTION_KEYWORD import com.jetbrains.snakecharm.lang.psi.elementTypes.SmkElementTypes import com.jetbrains.snakecharm.lang.psi.elementTypes.SmkStubElementTypes.* import java.util.* @@ -21,6 +23,7 @@ import java.util.* class SmkStatementParsing( context: SmkParserContext, ) : StatementParsing(context) { + val api = SnakemakeApiService.getInstance(context.project) // optional private val ruleSectionParsingData = SectionParsingData( declaration = RULE_DECLARATION_STATEMENT, @@ -75,7 +78,12 @@ class SmkStatementParsing( val scope = context.scope myBuilder.setDebugMode(false) - tryRemapCurrentToken(scope) { + // XXX: In order to support unrecognized top-level keywords Lexer return always Py:IDENTIFIER type + // Next we remap token to the appropriate token type if keyword is recognized or make a slower + // parsing as 'top-level' section attempt to check could it be parsed as top-level section + val toplevelArgsSectionsKeywords = api.getTopLevelArgsSectionsKeywords() // XXX: is optional, will work w/o it, but a bit slowly + tryRemapCurrentToken(scope, toplevelArgsSectionsKeywords) { + // try to parse as `section_name:` string and rollback to position before check checkIfAtToplevelDecoratorKeyword() } val tt = myBuilder.tokenType @@ -348,13 +356,21 @@ class SmkStatementParsing( return false } - private inline fun tryRemapCurrentToken(scope: SmkParsingScope, checkToplevelFun: () -> Boolean) { + private inline fun tryRemapCurrentToken( + scope: SmkParsingScope, + toplevelArgsSectionsKeywords: Set, + checkToplevelFun: () -> Boolean + ) { if (myBuilder.tokenType == PyTokenTypes.IDENTIFIER && !scope.inPythonicSection) { - val actualToken = SnakemakeLexer.KEYWORD_LIKE_SECTION_NAME_2_TOKEN_TYPE[myBuilder.tokenText!!] + val tokenText = myBuilder.tokenText!! + var actualToken = SnakemakeLexer.KEYWORD_LIKE_SECTION_NAME_2_TOKEN_TYPE[tokenText] + if (actualToken == null && tokenText in toplevelArgsSectionsKeywords) { + actualToken = WORKFLOW_TOPLEVEL_ARGS_SECTION_KEYWORD + } if (actualToken != null) { myBuilder.remapCurrentToken(actualToken) } else if (checkToplevelFun()) { - myBuilder.remapCurrentToken(SmkTokenTypes.WORKFLOW_TOPLEVEL_ARGS_SECTION_KEYWORD) + myBuilder.remapCurrentToken(WORKFLOW_TOPLEVEL_ARGS_SECTION_KEYWORD) } } } diff --git a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeLexer.kt b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeLexer.kt index fff90793b..c0d0a9c9b 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeLexer.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeLexer.kt @@ -82,7 +82,7 @@ class SnakemakeLexer : PythonIndentingLexer() { .add(SnakemakeNames.WORKFLOW_ONERROR_KEYWORD) .build() - private val SPECIAL_KEYWORDS_2_TOKEN_TYPE = ImmutableMap.Builder() + val SPECIAL_KEYWORDS_2_TOKEN_TYPE = ImmutableMap.Builder() .put(SnakemakeNames.RULE_KEYWORD, SmkTokenTypes.RULE_KEYWORD) .put(SnakemakeNames.CHECKPOINT_KEYWORD, SmkTokenTypes.CHECKPOINT_KEYWORD) .put(SnakemakeNames.WORKFLOW_LOCALRULES_KEYWORD, SmkTokenTypes.WORKFLOW_LOCALRULES_KEYWORD) diff --git a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeParser.kt b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeParser.kt index 692c2fea5..d22e3156b 100644 --- a/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeParser.kt +++ b/src/main/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeParser.kt @@ -1,6 +1,7 @@ package com.jetbrains.snakecharm.lang.parser import com.intellij.lang.SyntaxTreeBuilder +import com.intellij.openapi.project.Project import com.jetbrains.python.parsing.ParsingContext import com.jetbrains.python.parsing.PyParser import com.jetbrains.python.psi.LanguageLevel @@ -9,9 +10,9 @@ import com.jetbrains.python.psi.LanguageLevel * @author Roman.Chernyatchik * @date 2018-12-31 */ -class SnakemakeParser : PyParser() { +class SnakemakeParser(val project: Project) : PyParser() { override fun createParsingContext( builder: SyntaxTreeBuilder?, languageLevel: LanguageLevel? - ): ParsingContext = SmkParserContext(builder!!, languageLevel!!) + ): ParsingContext = SmkParserContext(builder!!, languageLevel!!, project) } \ No newline at end of file diff --git a/src/test/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeLexerTest.kt b/src/test/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeLexerTest.kt index 81672eece..1bd491050 100644 --- a/src/test/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeLexerTest.kt +++ b/src/test/kotlin/com/jetbrains/snakecharm/lang/parser/SnakemakeLexerTest.kt @@ -152,6 +152,25 @@ class SnakemakeLexerTest : PyLexerTestCase() { ) } + fun testToplevelKeywordsUnrecognized_Singleline() { + doTest( + """ + |fooo_booo_dooo: 1 + |""".trimMargin().trimStart(), + "Py:IDENTIFIER", "Py:COLON", "Py:SPACE", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:STATEMENT_BREAK" + ) + } + + fun testToplevelKeywordsUnrecognized_MultilpleLines() { + doTest( + """ + |fooo_booo_dooo: + | a=1 + |""".trimMargin().trimStart(), + "Py:IDENTIFIER", "Py:COLON", "Py:STATEMENT_BREAK", "Py:LINE_BREAK", "Py:INDENT", "Py:IDENTIFIER", "Py:EQ", "Py:INTEGER_LITERAL", "Py:STATEMENT_BREAK", "Py:DEDENT", "Py:LINE_BREAK", "Py:STATEMENT_BREAK" + ) + } + /*