diff --git a/doc/lexer-rules.md b/doc/lexer-rules.md index 6b4912118df..abc99c174d4 100644 --- a/doc/lexer-rules.md +++ b/doc/lexer-rules.md @@ -308,3 +308,16 @@ As of 4.5, you can also define channel names like enumerations with the followin ``` channels { WSCHANNEL, MYHIDDEN } ``` + +## Lexer Rule Options + +### caseInsensitive + +Defines if the current lexer rule is case-insensitive. +The argument can be `true`, `false`. +The option rewrites `caseInsensitive` grammar option value if it's defined. + +```g4 +options { caseInsensitive=true; } +STRING options { caseInsensitive=false; } : 'N'? '\'' (~'\'' | '\'\'')* '\''; // lower n is not allowed +``` \ No newline at end of file diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java index 81d1bc98ecd..9af6c6ec958 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestATNLexerInterpreter.java @@ -505,6 +505,26 @@ public void testSetUp() throws Exception { checkLexerMatches(lg, inputString, "TOKEN, EOF"); } + @Test public void testCaseInsensitiveInLexerRule() throws Exception { + LexerGrammar lg = new LexerGrammar( + "lexer grammar L;\n" + + "TOKEN1 options { caseInsensitive=true; } : [a-f]+;\n" + + "WS: [ ]+ -> skip;" + ); + + checkLexerMatches(lg, "ABCDEF", "TOKEN1, EOF"); + } + + @Test public void testCaseInsensitiveInLexerRuleOverridesGlobalValue() { + String grammar = + "lexer grammar L;\n" + + "options { caseInsensitive=true; }\n" + + "STRING options { caseInsensitive=false; } : 'N'? '\\'' (~'\\'' | '\\'\\'')* '\\'';\n"; + + execLexer("L.g4", grammar, "L", "n'sample'"); + assertEquals("line 1:0 token recognition error at: 'n'\n", getParseErrors()); + } + protected void checkLexerMatches(LexerGrammar lg, String inputString, String expecting) { ATN atn = createATN(lg, true); CharStream input = CharStreams.fromString(inputString); diff --git a/tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java b/tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java index 14ae5eff61e..81075bd65b8 100644 --- a/tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java +++ b/tool-testsuite/test/org/antlr/v4/test/tool/TestSymbolIssues.java @@ -472,13 +472,45 @@ public void testLabelsForTokensWithMixedTypesLRWithoutLabels() { testErrors(test, false); } - @Test public void testIllegalModeOption() { + @Test public void testIllegalCaseInsensitiveOptionValue() { String[] test = { "lexer grammar L;\n" + "options { caseInsensitive = badValue; }\n" + - "DEFAULT_TOKEN: [A-F]+;\n", + "TOKEN_1 options { caseInsensitive = badValue; } : [A-F]+;\n", - "warning(" + ErrorType.ILLEGAL_OPTION_VALUE.code + "): L.g4:2:28: unsupported option value caseInsensitive=badValue\n" + "warning(" + ErrorType.ILLEGAL_OPTION_VALUE.code + "): L.g4:2:28: unsupported option value caseInsensitive=badValue\n" + + "warning(" + ErrorType.ILLEGAL_OPTION_VALUE.code + "): L.g4:3:36: unsupported option value caseInsensitive=badValue\n" + }; + + testErrors(test, false); + } + + @Test public void testRedundantCaseInsensitiveLexerRuleOption() { + String[] test = { + "lexer grammar L;\n" + + "options { caseInsensitive = true; }\n" + + "TOKEN options { caseInsensitive = true; } : [A-F]+;\n", + + "warning(" + ErrorType.REDUNDANT_CASE_INSENSITIVE_LEXER_RULE_OPTION.code + "): L.g4:3:16: caseInsensitive lexer rule option is redundant because its value equals to global value (true)\n" + }; + testErrors(test, false); + + String[] test2 = { + "lexer grammar L;\n" + + "options { caseInsensitive = false; }\n" + + "TOKEN options { caseInsensitive = false; } : [A-F]+;\n", + + "warning(" + ErrorType.REDUNDANT_CASE_INSENSITIVE_LEXER_RULE_OPTION.code + "): L.g4:3:16: caseInsensitive lexer rule option is redundant because its value equals to global value (false)\n" + }; + testErrors(test2, false); + } + + @Test public void testCaseInsensitiveOptionInParseRule() { + String[] test = { + "grammar G;\n" + + "root options { caseInsensitive=true; } : 'token';", + + "warning(" + ErrorType.ILLEGAL_OPTION.code + "): G.g4:2:15: unsupported option caseInsensitive\n" }; testErrors(test, false); diff --git a/tool/src/org/antlr/v4/Tool.java b/tool/src/org/antlr/v4/Tool.java index d2c95f525de..3ac543652a5 100644 --- a/tool/src/org/antlr/v4/Tool.java +++ b/tool/src/org/antlr/v4/Tool.java @@ -288,8 +288,8 @@ protected void handleOptionSetArg(String arg) { errMgr.toolError(ErrorType.BAD_OPTION_SET_SYNTAX, arg); return; } - if ( Grammar.parserOptions.contains(option) || - Grammar.lexerOptions.contains(option) ) + if ( Grammar.ParserOptions.contains(option) || + Grammar.LexerOptions.contains(option) ) { if ( grammarOptions==null ) grammarOptions = new HashMap(); grammarOptions.put(option, value); diff --git a/tool/src/org/antlr/v4/automata/LexerATNFactory.java b/tool/src/org/antlr/v4/automata/LexerATNFactory.java index 5f39325d670..b3427de1249 100644 --- a/tool/src/org/antlr/v4/automata/LexerATNFactory.java +++ b/tool/src/org/antlr/v4/automata/LexerATNFactory.java @@ -77,8 +77,6 @@ public class LexerATNFactory extends ParserATNFactory { private final List ruleCommands = new ArrayList(); - private final boolean caseInsensitive; - /** * Maps from an action index to a {@link LexerAction} object. */ @@ -92,8 +90,6 @@ public LexerATNFactory(LexerGrammar g) { super(g); // use codegen to get correct language templates for lexer commands String language = g.getOptionString("language"); - String caseInsensitiveOption = g.getOptionString("caseInsensitive"); - caseInsensitive = caseInsensitiveOption != null && caseInsensitiveOption.equals("true"); CodeGenerator gen = new CodeGenerator(g.tool, null, language); codegenTemplates = gen.getTemplates(); } @@ -207,51 +203,42 @@ public Handle lexerAltCommands(Handle alt, Handle cmds) { @Override public Handle lexerCallCommand(GrammarAST ID, GrammarAST arg) { - LexerAction lexerAction = createLexerAction(ID, arg); - if (lexerAction != null) { - return action(ID, lexerAction); - } - - // fall back to standard action generation for the command - ST cmdST = codegenTemplates.getInstanceOf("Lexer" + - CharSupport.capitalize(ID.getText())+ - "Command"); - if (cmdST == null) { - g.tool.errMgr.grammarError(ErrorType.INVALID_LEXER_COMMAND, g.fileName, ID.token, ID.getText()); - return epsilon(ID); - } - - if (cmdST.impl.formalArguments == null || !cmdST.impl.formalArguments.containsKey("arg")) { - g.tool.errMgr.grammarError(ErrorType.UNWANTED_LEXER_COMMAND_ARGUMENT, g.fileName, ID.token, ID.getText()); - return epsilon(ID); - } - - cmdST.add("arg", arg.getText()); - cmdST.add("grammar", arg.g); - return action(cmdST.render()); + return lexerCallCommandOrCommand(ID, arg); } @Override public Handle lexerCommand(GrammarAST ID) { - LexerAction lexerAction = createLexerAction(ID, null); + return lexerCallCommandOrCommand(ID, null); + } + + private Handle lexerCallCommandOrCommand(GrammarAST ID, GrammarAST arg) { + LexerAction lexerAction = createLexerAction(ID, arg); if (lexerAction != null) { return action(ID, lexerAction); } // fall back to standard action generation for the command ST cmdST = codegenTemplates.getInstanceOf("Lexer" + - CharSupport.capitalize(ID.getText()) + + CharSupport.capitalize(ID.getText())+ "Command"); if (cmdST == null) { g.tool.errMgr.grammarError(ErrorType.INVALID_LEXER_COMMAND, g.fileName, ID.token, ID.getText()); return epsilon(ID); } - if (cmdST.impl.formalArguments != null && cmdST.impl.formalArguments.containsKey("arg")) { - g.tool.errMgr.grammarError(ErrorType.MISSING_LEXER_COMMAND_ARGUMENT, g.fileName, ID.token, ID.getText()); + boolean callCommand = arg != null; + boolean containsArg = cmdST.impl.formalArguments != null && cmdST.impl.formalArguments.containsKey("arg"); + if (callCommand != containsArg) { + ErrorType errorType = callCommand ? ErrorType.UNWANTED_LEXER_COMMAND_ARGUMENT : ErrorType.MISSING_LEXER_COMMAND_ARGUMENT; + g.tool.errMgr.grammarError(errorType, g.fileName, ID.token, ID.getText()); return epsilon(ID); } + if (callCommand) { + cmdST.add("arg", arg.getText()); + cmdST.add("grammar", arg.g); + } + return action(cmdST.render()); } @@ -279,7 +266,7 @@ public Handle set(GrammarAST associatedAST, List alts, boolean inver int a = CharSupport.getCharValueFromGrammarCharLiteral(t.getChild(0).getText()); int b = CharSupport.getCharValueFromGrammarCharLiteral(t.getChild(1).getText()); if (checkRange((GrammarAST)t.getChild(0), (GrammarAST)t.getChild(1), a, b)) { - checkRangeAndAddToSet(associatedAST, t, set, a, b, caseInsensitive, null); + checkRangeAndAddToSet(associatedAST, t, set, a, b, currentRule.caseInsensitive, null); } } else if ( t.getType()==ANTLRParser.LEXER_CHAR_SET ) { @@ -567,11 +554,11 @@ private void applyPrevState(GrammarAST charSetAST, IntervalSet set, CharSetParse } private void checkCharAndAddToSet(GrammarAST ast, IntervalSet set, int c) { - checkRangeAndAddToSet(ast, ast, set, c, c, caseInsensitive, null); + checkRangeAndAddToSet(ast, ast, set, c, c, currentRule.caseInsensitive, null); } private void checkRangeAndAddToSet(GrammarAST mainAst, IntervalSet set, int a, int b) { - checkRangeAndAddToSet(mainAst, mainAst, set, a, b, caseInsensitive, null); + checkRangeAndAddToSet(mainAst, mainAst, set, a, b, currentRule.caseInsensitive, null); } private CharactersDataCheckStatus checkRangeAndAddToSet(GrammarAST rootAst, GrammarAST ast, IntervalSet set, int a, int b, boolean caseInsensitive, CharactersDataCheckStatus previousStatus) { @@ -630,7 +617,7 @@ private CharactersDataCheckStatus checkRangeAndAddToSet(GrammarAST rootAst, Gram private Transition createTransition(ATNState target, int from, int to, CommonTree tree) { RangeBorderCharactersData charactersData = RangeBorderCharactersData.getAndCheckCharactersData(from, to, g, tree, true); - if (caseInsensitive) { + if (currentRule.caseInsensitive) { if (charactersData.isSingleRange()) { return CodePointTransitions.createWithCodePointRange(target, from, to); } diff --git a/tool/src/org/antlr/v4/parse/ANTLRLexer.g b/tool/src/org/antlr/v4/parse/ANTLRLexer.g index ff127d5fbef..cf024c57be8 100644 --- a/tool/src/org/antlr/v4/parse/ANTLRLexer.g +++ b/tool/src/org/antlr/v4/parse/ANTLRLexer.g @@ -163,14 +163,26 @@ import org.antlr.v4.runtime.misc.Interval; * Return token or null if for some reason we can't find the start. */ public Token getRuleOrSubruleStartToken() { - if ( tokens==null ) return null; + if (tokens == null) return null; int i = tokens.index(); - int n = tokens.size(); - if ( i>=n ) i = n-1; // seems index == n as we lex - while ( i>=0 && i= n) i = n - 1; // seems index == n as we lex + boolean withinOptionsBlock = false; + while (i >= 0 && i < n) { int ttype = tokens.get(i).getType(); - if ( ttype == LPAREN || ttype == TOKEN_REF || ttype == RULE_REF ) { - return tokens.get(i); + if (withinOptionsBlock) { + // Ignore rule options content + if (ttype == OPTIONS) { + withinOptionsBlock = false; + } + } + else { + if (ttype == RBRACE) { + withinOptionsBlock = true; + } + else if (ttype == LPAREN || ttype == TOKEN_REF || ttype == RULE_REF) { + return tokens.get(i); + } } i--; } diff --git a/tool/src/org/antlr/v4/parse/ANTLRParser.g b/tool/src/org/antlr/v4/parse/ANTLRParser.g index 9b4866521de..bd6861ac92c 100644 --- a/tool/src/org/antlr/v4/parse/ANTLRParser.g +++ b/tool/src/org/antlr/v4/parse/ANTLRParser.g @@ -520,9 +520,13 @@ lexerRule paraphrases.pop(); } : FRAGMENT? - TOKEN_REF COLON lexerRuleBlock SEMI + TOKEN_REF + + optionsSpec? + + COLON lexerRuleBlock SEMI -> ^( RULE TOKEN_REF - ^(RULEMODIFIERS FRAGMENT)? lexerRuleBlock + ^(RULEMODIFIERS FRAGMENT)? optionsSpec? lexerRuleBlock ) ; diff --git a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g index 9bc399641e7..7bac2c42e0c 100644 --- a/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g +++ b/tool/src/org/antlr/v4/parse/GrammarTreeVisitor.g @@ -144,7 +144,7 @@ public void discoverRule(RuleAST rule, GrammarAST ID, List modifiers List actions, GrammarAST block) { } public void finishRule(RuleAST rule, GrammarAST ID, GrammarAST block) { } -public void discoverLexerRule(RuleAST rule, GrammarAST ID, List modifiers, +public void discoverLexerRule(RuleAST rule, GrammarAST ID, List modifiers, GrammarAST options, GrammarAST block) { } public void finishLexerRule(RuleAST rule, GrammarAST ID, GrammarAST block) { } public void ruleCatch(GrammarAST arg, ActionAST action) { } @@ -525,7 +525,8 @@ lexerRule : ^( RULE TOKEN_REF {currentRuleName=$TOKEN_REF.text; currentRuleAST=$RULE;} (^(RULEMODIFIERS m=FRAGMENT {mods.add($m);}))? - {discoverLexerRule((RuleAST)$RULE, $TOKEN_REF, mods, (GrammarAST)input.LT(1));} + opts=optionsSpec* + {discoverLexerRule((RuleAST)$RULE, $TOKEN_REF, mods, $opts.start, (GrammarAST)input.LT(1));} lexerRuleBlock { finishLexerRule((RuleAST)$RULE, $TOKEN_REF, $lexerRuleBlock.start); diff --git a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java index 17ff09e1479..8d737df793e 100644 --- a/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java +++ b/tool/src/org/antlr/v4/semantics/BasicSemanticChecks.java @@ -30,6 +30,7 @@ import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** No side-effects except for setting options into the appropriate node. * TODO: make the side effects into a separate pass this @@ -100,6 +101,11 @@ public class BasicSemanticChecks extends GrammarTreeVisitor { */ private boolean inFragmentRule; + /** + * Value of caseInsensitive option (false if not defined) + */ + private boolean grammarCaseInsensitive = false; + public BasicSemanticChecks(Grammar g, RuleCollector ruleCollector) { this.g = g; this.ruleCollector = ruleCollector; @@ -182,7 +188,7 @@ public void discoverRule(RuleAST rule, GrammarAST ID, } @Override - public void discoverLexerRule(RuleAST rule, GrammarAST ID, List modifiers, + public void discoverLexerRule(RuleAST rule, GrammarAST ID, List modifiers, GrammarAST options, GrammarAST block) { checkInvalidRuleDef(ID.token); @@ -210,6 +216,11 @@ public void ruleRef(GrammarAST ref, ActionAST arg) { checkInvalidRuleRef(ref.token); } + @Override + public void grammarOption(GrammarAST ID, GrammarAST valueAST) { + checkOptions(g.ast, ID.token, valueAST); + } + @Override public void ruleOption(GrammarAST ID, GrammarAST valueAST) { checkOptions((GrammarAST)ID.getAncestor(RULE), ID.token, valueAST); @@ -220,19 +231,6 @@ public void blockOption(GrammarAST ID, GrammarAST valueAST) { checkOptions((GrammarAST)ID.getAncestor(BLOCK), ID.token, valueAST); } - @Override - public void grammarOption(GrammarAST ID, GrammarAST valueAST) { - boolean ok = checkOptions(g.ast, ID.token, valueAST); - //if ( ok ) g.ast.setOption(ID.getText(), value); - if (ID.getText().equals("caseInsensitive")) { - String valueText = valueAST.getText(); - if (!valueText.equals("true") && !valueText.equals("false")) { - g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION_VALUE, g.fileName, valueAST.getToken(), - ID.getText(), valueText); - } - } - } - @Override public void defineToken(GrammarAST ID) { checkTokenDefinition(ID.token); @@ -240,11 +238,13 @@ public void defineToken(GrammarAST ID) { @Override protected void enterChannelsSpec(GrammarAST tree) { - if (g.isParser()) { - g.tool.errMgr.grammarError(ErrorType.CHANNELS_BLOCK_IN_PARSER_GRAMMAR, g.fileName, tree.token); - } - else if (g.isCombined()) { - g.tool.errMgr.grammarError(ErrorType.CHANNELS_BLOCK_IN_COMBINED_GRAMMAR, g.fileName, tree.token); + ErrorType errorType = g.isParser() + ? ErrorType.CHANNELS_BLOCK_IN_PARSER_GRAMMAR + : g.isCombined() + ? ErrorType.CHANNELS_BLOCK_IN_COMBINED_GRAMMAR + : null; + if (errorType != null) { + g.tool.errMgr.grammarError(errorType, g.fileName, tree.token); } } @@ -255,16 +255,7 @@ public void defineChannel(GrammarAST ID) { @Override public void elementOption(GrammarASTWithOptions elem, GrammarAST ID, GrammarAST valueAST) { - String v = null; - boolean ok = checkElementOptions(elem, ID, valueAST); -// if ( ok ) { -// if ( v!=null ) { -// t.setOption(ID.getText(), v); -// } -// else { -// t.setOption(TerminalAST.defaultTokenOption, v); -// } -// } + checkElementOptions(elem, ID, valueAST); } @Override @@ -483,46 +474,52 @@ protected void enterTerminal(GrammarAST tree) { } /** Check option is appropriate for grammar, rule, subrule */ - boolean checkOptions(GrammarAST parent, - Token optionID, - GrammarAST valueAST) - { - boolean ok = true; - if ( parent.getType()==ANTLRParser.BLOCK ) { - if ( g.isLexer() && !Grammar.LexerBlockOptions.contains(optionID.getText()) ) { // block - g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, - g.fileName, - optionID, - optionID.getText()); - ok = false; - } - if ( !g.isLexer() && !Grammar.ParserBlockOptions.contains(optionID.getText()) ) { // block - g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, - g.fileName, - optionID, - optionID.getText()); - ok = false; + void checkOptions(GrammarAST parent, Token optionID, GrammarAST valueAST) { + Set optionsToCheck = null; + int parentType = parent.getType(); + switch (parentType) { + case ANTLRParser.BLOCK: + optionsToCheck = g.isLexer() ? Grammar.LexerBlockOptions : Grammar.ParserBlockOptions; + break; + case ANTLRParser.RULE: + optionsToCheck = g.isLexer() ? Grammar.LexerRuleOptions : Grammar.ParseRuleOptions; + break; + case ANTLRParser.GRAMMAR: + optionsToCheck = g.getType() == ANTLRParser.LEXER + ? Grammar.LexerOptions + : Grammar.ParserOptions; + break; + } + String optionName = optionID.getText(); + if (optionsToCheck != null && !optionsToCheck.contains(optionName)) { + g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, g.fileName, optionID, optionName); + } + else { + checkCaseInsensitiveOption(optionID, valueAST, parentType); + } + } + + private void checkCaseInsensitiveOption(Token optionID, GrammarAST valueAST, int parentType) { + String optionName = optionID.getText(); + if (optionName.equals(Grammar.caseInsensitiveOptionName)) { + String valueText = valueAST.getText(); + if (valueText.equals("true") || valueText.equals("false")) { + boolean currentValue = Boolean.parseBoolean(valueText); + if (parentType == ANTLRParser.GRAMMAR) { + grammarCaseInsensitive = currentValue; + } + else { + if (grammarCaseInsensitive == currentValue) { + g.tool.errMgr.grammarError(ErrorType.REDUNDANT_CASE_INSENSITIVE_LEXER_RULE_OPTION, + g.fileName, optionID, currentValue); + } + } } - } - else if ( parent.getType()==ANTLRParser.RULE ) { - if ( !Grammar.ruleOptions.contains(optionID.getText()) ) { // rule - g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, - g.fileName, - optionID, - optionID.getText()); - ok = false; + else { + g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION_VALUE, g.fileName, valueAST.getToken(), + optionName, valueText); } } - else if ( parent.getType()==ANTLRParser.GRAMMAR && - !legalGrammarOption(optionID.getText()) ) { // grammar - g.tool.errMgr.grammarError(ErrorType.ILLEGAL_OPTION, - g.fileName, - optionID, - optionID.getText()); - ok = false; - } - - return ok; } /** Check option is appropriate for elem; parent of ID is ELEMENT_OPTIONS */ @@ -594,17 +591,6 @@ boolean checkTokenOptions(TerminalAST elem, GrammarAST ID, GrammarAST valueAST) return true; } - boolean legalGrammarOption(String key) { - switch ( g.getType() ) { - case ANTLRParser.LEXER : - return Grammar.lexerOptions.contains(key); - case ANTLRParser.PARSER : - return Grammar.parserOptions.contains(key); - default : - return Grammar.parserOptions.contains(key); - } - } - void checkImport(Token importID) { Grammar delegate = g.getImportedGrammar(importID.getText()); if ( delegate==null ) return; diff --git a/tool/src/org/antlr/v4/semantics/RuleCollector.java b/tool/src/org/antlr/v4/semantics/RuleCollector.java index 0931986451a..a547120425e 100644 --- a/tool/src/org/antlr/v4/semantics/RuleCollector.java +++ b/tool/src/org/antlr/v4/semantics/RuleCollector.java @@ -11,11 +11,7 @@ import org.antlr.v4.misc.Utils; import org.antlr.v4.parse.GrammarTreeVisitor; import org.antlr.v4.parse.ScopeParser; -import org.antlr.v4.tool.AttributeDict; -import org.antlr.v4.tool.ErrorManager; -import org.antlr.v4.tool.Grammar; -import org.antlr.v4.tool.LeftRecursiveRule; -import org.antlr.v4.tool.Rule; +import org.antlr.v4.tool.*; import org.antlr.v4.tool.ast.ActionAST; import org.antlr.v4.tool.ast.AltAST; import org.antlr.v4.tool.ast.GrammarAST; @@ -27,6 +23,8 @@ import java.util.Map; public class RuleCollector extends GrammarTreeVisitor { + private boolean grammarCaseInsensitive = false; + /** which grammar are we checking */ public Grammar g; public ErrorManager errMgr; @@ -101,14 +99,43 @@ public void discoverOuterAlt(AltAST alt) { } } + @Override + public void grammarOption(GrammarAST ID, GrammarAST valueAST) { + Boolean caseInsensitive = getCaseInsensitiveValue(ID, valueAST); + if (caseInsensitive != null) { + grammarCaseInsensitive = caseInsensitive; + } + } + @Override public void discoverLexerRule(RuleAST rule, GrammarAST ID, List modifiers, - GrammarAST block) + GrammarAST options, GrammarAST block) { + boolean currentCaseInsensitive = grammarCaseInsensitive; + if (options != null) { + for (Object child : options.getChildren()) { + GrammarAST childAST = (GrammarAST) child; + Boolean caseInsensitive = getCaseInsensitiveValue((GrammarAST)childAST.getChild(0), (GrammarAST)childAST.getChild(1)); + if (caseInsensitive != null) { + currentCaseInsensitive = caseInsensitive; + } + } + } + int numAlts = block.getChildCount(); - Rule r = new Rule(g, ID.getText(), rule, numAlts); - r.mode = currentModeName; + Rule r = new Rule(g, ID.getText(), rule, numAlts, currentModeName, currentCaseInsensitive); if ( !modifiers.isEmpty() ) r.modifiers = modifiers; rules.put(r.name, r); } + + private Boolean getCaseInsensitiveValue(GrammarAST optionID, GrammarAST valueAST) { + String optionName = optionID.getText(); + if (optionName.equals(Grammar.caseInsensitiveOptionName)) { + String valueText = valueAST.getText(); + if (valueText.equals("true") || valueText.equals("false")) { + return Boolean.parseBoolean(valueText); + } + } + return null; + } } diff --git a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java index dc3cb2fb001..07e142d4f2d 100644 --- a/tool/src/org/antlr/v4/semantics/SemanticPipeline.java +++ b/tool/src/org/antlr/v4/semantics/SemanticPipeline.java @@ -177,7 +177,6 @@ void assignLexerTokenTypes(Grammar g, List tokensDefs) { } } } - } boolean hasTypeOrMoreCommand(Rule r) { diff --git a/tool/src/org/antlr/v4/semantics/SymbolCollector.java b/tool/src/org/antlr/v4/semantics/SymbolCollector.java index 7c41eef1d3d..66f8cac88ec 100644 --- a/tool/src/org/antlr/v4/semantics/SymbolCollector.java +++ b/tool/src/org/antlr/v4/semantics/SymbolCollector.java @@ -91,7 +91,7 @@ public void discoverRule(RuleAST rule, GrammarAST ID, } @Override - public void discoverLexerRule(RuleAST rule, GrammarAST ID, List modifiers, + public void discoverLexerRule(RuleAST rule, GrammarAST ID, List modifiers, GrammarAST options, GrammarAST block) { currentRule = g.getRule(ID.getText()); diff --git a/tool/src/org/antlr/v4/tool/ErrorType.java b/tool/src/org/antlr/v4/tool/ErrorType.java index a0f374b3338..a1e0683bd7c 100644 --- a/tool/src/org/antlr/v4/tool/ErrorType.java +++ b/tool/src/org/antlr/v4/tool/ErrorType.java @@ -1107,6 +1107,19 @@ public enum ErrorType { "Range .. probably contains not implied characters . Both bounds should be defined in lower or UPPER case", ErrorSeverity.WARNING ), + /** + *

Redundant caseInsensitive lexer rule option

+ * + * 
+	 * options { caseInsensitive=true; }
+	 * TOKEN options { caseInsensitive=true; } : [a-z]+ -> caseInsensitive(true); // warning
+	 *
+ */ + REDUNDANT_CASE_INSENSITIVE_LEXER_RULE_OPTION( + 186, + "caseInsensitive lexer rule option is redundant because its value equals to global value ()", + ErrorSeverity.WARNING + ), /* * Backward incompatibility errors diff --git a/tool/src/org/antlr/v4/tool/Grammar.java b/tool/src/org/antlr/v4/tool/Grammar.java index cbd4af76add..36fb49f4f6f 100644 --- a/tool/src/org/antlr/v4/tool/Grammar.java +++ b/tool/src/org/antlr/v4/tool/Grammar.java @@ -74,21 +74,28 @@ public class Grammar implements AttributeResolver { */ public static final String INVALID_RULE_NAME = ""; - public static final Set parserOptions = new HashSet(); + public static final String caseInsensitiveOptionName = "caseInsensitive"; + + public static final Set ParserOptions = new HashSet(); static { - parserOptions.add("superClass"); - parserOptions.add("contextSuperClass"); - parserOptions.add("TokenLabelType"); - parserOptions.add("tokenVocab"); - parserOptions.add("language"); - parserOptions.add("accessLevel"); - parserOptions.add("exportMacro"); - parserOptions.add("caseInsensitive"); + ParserOptions.add("superClass"); + ParserOptions.add("contextSuperClass"); + ParserOptions.add("TokenLabelType"); + ParserOptions.add("tokenVocab"); + ParserOptions.add("language"); + ParserOptions.add("accessLevel"); + ParserOptions.add("exportMacro"); + ParserOptions.add(caseInsensitiveOptionName); } - public static final Set lexerOptions = parserOptions; + public static final Set LexerOptions = ParserOptions; + + public static final Set LexerRuleOptions = new HashSet<>(); + static { + LexerRuleOptions.add(caseInsensitiveOptionName); + } - public static final Set ruleOptions = new HashSet(); + public static final Set ParseRuleOptions = new HashSet<>(); public static final Set ParserBlockOptions = new HashSet(); diff --git a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java index f6e4cdfbf2f..79801f1a967 100644 --- a/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java +++ b/tool/src/org/antlr/v4/tool/GrammarTransformPipeline.java @@ -276,7 +276,7 @@ public void integrateImportedGrammars(Grammar rootGrammar) { } // COPY MODES - // The strategy is to copy all the mode sections rules across to any + // The strategy is to copy all the mode sections rules across to any // mode section in the new grammar with the same name or a new // mode section if no matching mode is resolved. Rules which are // already in the new grammar are ignored for copy. If the mode @@ -311,7 +311,7 @@ public void integrateImportedGrammars(Grammar rootGrammar) { destinationAST.addChild(r); addedRules++; rootRuleNames.add(ruleName); - } + } } if (!rootAlreadyHasMode && addedRules > 0) { rootGrammar.ast.addChild(destinationAST); @@ -405,7 +405,7 @@ public GrammarRootAST extractImplicitLexer(Grammar combinedGrammar) { GrammarAST[] options = optionsRoot.getChildren().toArray(new GrammarAST[0]); for (GrammarAST o : options) { String optionName = o.getChild(0).getText(); - if ( Grammar.lexerOptions.contains(optionName) && + if ( Grammar.LexerOptions.contains(optionName) && !Grammar.doNotCopyOptionsToLexer.contains(optionName) ) { GrammarAST optionTree = (GrammarAST)adaptor.dupTree(o); diff --git a/tool/src/org/antlr/v4/tool/Rule.java b/tool/src/org/antlr/v4/tool/Rule.java index 401dfaa60f2..5389f2b182e 100644 --- a/tool/src/org/antlr/v4/tool/Rule.java +++ b/tool/src/org/antlr/v4/tool/Rule.java @@ -64,7 +64,10 @@ public class Rule implements AttributeResolver { public Grammar g; /** If we're in a lexer grammar, we might be in a mode */ - public String mode; + public final String mode; + + /** If null then use value from global option that is false by default */ + public final boolean caseInsensitive; /** Map a name to an action for this rule like @init {...}. * The code generator will use this to fill holes in the rule template. @@ -103,12 +106,18 @@ public class Rule implements AttributeResolver { public int actionIndex = -1; // if lexer; 0..n-1 for n actions in a rule public Rule(Grammar g, String name, RuleAST ast, int numberOfAlts) { + this(g, name, ast, numberOfAlts, null, false); + } + + public Rule(Grammar g, String name, RuleAST ast, int numberOfAlts, String lexerMode, boolean caseInsensitive) { this.g = g; this.name = name; this.ast = ast; this.numberOfAlts = numberOfAlts; alt = new Alternative[numberOfAlts+1]; // 1..n for (int i=1; i<=numberOfAlts; i++) alt[i] = new Alternative(this, i); + this.mode = lexerMode; + this.caseInsensitive = caseInsensitive; } public void defineActionInAlt(int currentAlt, ActionAST actionAST) {