Skip to content

Commit

Permalink
Allow StringConcat to start with hex literal (#360)
Browse files Browse the repository at this point in the history 
fixes #359
  • Loading branch information
@MarkusAmshove
MarkusAmshove authored Sep 9, 2023
1 parent 2910f83 commit 6c9ca4f
Show file tree
Hide file tree
Showing 3 changed files with 70 additions and 93 deletions.
21 changes: 6 additions & 15 deletions libs/natparse/src/main/java/org/amshove/natparse/parsing/AbstractParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,6 @@ public AbstractParser(IModuleProvider moduleProvider)
this.moduleProvider = moduleProvider;
}

void setModuleProvider(IModuleProvider moduleProvider)
{
this.moduleProvider = moduleProvider;
}

public ParseResult<T> parse(TokenList tokens)
{
this.tokens = tokens;
Expand Down Expand Up @@ -225,20 +220,18 @@ protected SyntaxToken consumeMandatory(BaseSyntaxNode node, SyntaxKind kind) thr
throw new ParseError(peek());
}

protected SyntaxToken consumeMandatoryClosing(BaseSyntaxNode node, SyntaxKind closingTokenType, SyntaxToken openingToken) throws ParseError
protected void consumeMandatoryClosing(BaseSyntaxNode node, SyntaxKind closingTokenType, SyntaxToken openingToken) throws ParseError
{
if (peekKind(SyntaxKind.END_ALL) && END_KINDS_THAT_END_ALL_ENDS.contains(closingTokenType)) // sort
{
return peek();
return;
}

if (!consumeOptionally(node, closingTokenType))
{
diagnostics.add(ParserErrors.missingClosingToken(closingTokenType, openingToken));
throw new ParseError(peek());
}

return previousToken();
}

protected IOperandNode consumeLiteralNode(BaseSyntaxNode node) throws ParseError
Expand Down Expand Up @@ -268,7 +261,7 @@ protected IOperandNode consumeLiteralNode(BaseSyntaxNode node) throws ParseError
return literal;
}

if (peekKind(SyntaxKind.STRING_LITERAL) && peekKind(1, SyntaxKind.MINUS))
if (peekAny(List.of(SyntaxKind.STRING_LITERAL, SyntaxKind.HEX_LITERAL)) && peekKind(1, SyntaxKind.MINUS))
{
return consumeStringConcat(node);
}
Expand Down Expand Up @@ -328,31 +321,29 @@ protected IOperandNode consumeLiteralNode(BaseSyntaxNode node, SyntaxKind litera
return literal;
}

protected SyntaxToken consumeLiteralToken(BaseSyntaxNode node) throws ParseError
protected void consumeLiteralToken(BaseSyntaxNode node) throws ParseError
{
if (peek().kind().isSystemVariable())
{
var systemVariable = peek();
node.addNode(new SystemVariableNode(systemVariable));
discard();
return systemVariable;
return;
}

if (peek().kind() == SyntaxKind.LPAREN) // Attributes
{
var lparen = peek(); // TODO(attributes): This is not correct but good for now.
while (!isAtEnd() && peek().kind() != SyntaxKind.RPAREN && peek().kind() != SyntaxKind.END_DEFINE)
{
consume(node);
}
consumeMandatory(node, SyntaxKind.RPAREN);
return lparen;
return;
}

var literal = consumeAny(List.of(SyntaxKind.NUMBER_LITERAL, SyntaxKind.STRING_LITERAL, SyntaxKind.HEX_LITERAL, SyntaxKind.TRUE, SyntaxKind.FALSE, SyntaxKind.DATE_LITERAL, SyntaxKind.TIME_LITERAL, SyntaxKind.EXTENDED_TIME_LITERAL));
previousNode = new TokenNode(literal);
node.addNode(previousNode);
return literal;
}

protected ITokenNode consumeMandatoryIdentifierTokenNode(BaseSyntaxNode node) throws ParseError
Expand Down
93 changes: 29 additions & 64 deletions libs/natparse/src/main/java/org/amshove/natparse/parsing/DefineDataParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ private void checkGroupIsNotEmpty(GroupNode groupNode)
{
return;
}
if (groupNode.variables().size() == 0)
if (groupNode.variables().isEmpty())
{
report(ParserErrors.emptyGroupVariable(groupNode));
}
Expand Down Expand Up @@ -151,7 +151,7 @@ private ScopeNode scope() throws ParseError
if (peekKind(SyntaxKind.BLOCK))
{
/*var block = */
block(); // TODO: Maybe do something with block
scopeNode.addNode(block());
}

var variable = variable(currentGroupsDimensions());
Expand Down Expand Up @@ -719,21 +719,9 @@ private void checkVariableType(TypedVariableNode variable)
{
switch (variable.type().format())
{
case ALPHANUMERIC:
case BINARY:
case UNICODE:
break;

case CONTROL:
case DATE:
case FLOAT:
case INTEGER:
case LOGIC:
case NUMERIC:
case PACKED:
case TIME:
case NONE:
report(ParserErrors.dynamicVariableLengthNotAllowed(variable));
case ALPHANUMERIC, BINARY, UNICODE ->
{}
case CONTROL, DATE, FLOAT, INTEGER, LOGIC, NUMERIC, PACKED, TIME, NONE -> report(ParserErrors.dynamicVariableLengthNotAllowed(variable));
}

if (variableLength > 0.0)
Expand Down Expand Up @@ -785,27 +773,16 @@ private void checkVariableType(TypedVariableNode variable)
{
switch (variable.type().format())
{
case ALPHANUMERIC:
case BINARY:
case UNICODE:
case ALPHANUMERIC, BINARY, UNICODE ->
{
if (!variable.type().hasDynamicLength())
{
report(ParserErrors.dataTypeNeedsLength(variable));
}
break;

case CONTROL:
case DATE:
case LOGIC:
case TIME:
case NONE:
break;

case FLOAT:
case INTEGER:
case NUMERIC:
case PACKED:
report(ParserErrors.dataTypeNeedsLength(variable));
}
case CONTROL, DATE, LOGIC, TIME, NONE ->
{}
case FLOAT, INTEGER, NUMERIC, PACKED -> report(ParserErrors.dataTypeNeedsLength(variable));
}
}

Expand All @@ -815,39 +792,25 @@ private void checkVariableType(TypedVariableNode variable)

switch (variable.type().format())
{
case ALPHANUMERIC:
case ALPHANUMERIC ->
{
if (initialValueKind.isBoolean())
{
break;
}
expectInitialValueType(variable, initialValueKind, SyntaxKind.STRING_LITERAL, SyntaxKind.NUMBER_LITERAL, SyntaxKind.HEX_LITERAL);
break;

case DATE:
expectInitialValueType(variable, initialValueKind, SyntaxKind.DATE_LITERAL);
break;

case BINARY:
case CONTROL:
case TIME:
case UNICODE:
case NONE:
// TODO: Unsure about these at the moment
break;

case FLOAT:
case NUMERIC:
case PACKED:
case INTEGER:
expectInitialValueType(variable, initialValueKind, SyntaxKind.NUMBER_LITERAL);
break;

case LOGIC:
}
case DATE -> expectInitialValueType(variable, initialValueKind, SyntaxKind.DATE_LITERAL);
case BINARY, CONTROL, TIME, UNICODE, NONE ->
{}
case FLOAT, NUMERIC, PACKED, INTEGER -> expectInitialValueType(variable, initialValueKind, SyntaxKind.NUMBER_LITERAL);
case LOGIC ->
{
if (initialValueKind != SyntaxKind.TRUE && initialValueKind != SyntaxKind.FALSE)
{
report(ParserErrors.initValueMismatch(variable, SyntaxKind.TRUE, SyntaxKind.FALSE));
}
break;
}
}
}
}
Expand Down Expand Up @@ -942,12 +905,12 @@ private int extractArrayBound(ITokenNode token, ArrayDimension dimension)
{
var isUnboundV = token.token().symbolName().equals("V"); // (1:V) is allowed in parameter scope, where V stands for variable

if (currentScope.isParameter() && isUnboundV && !isVariableDeclared(token.token().symbolName()))
if (currentScope.isParameter() && isUnboundV && isVariableUndeclared(token.token().symbolName()))
{
return IArrayDimension.VARIABLE_BOUND;
}

if (!isVariableDeclared(token.token().symbolName()))
if (isVariableUndeclared(token.token().symbolName()))
{
report(ParserErrors.unresolvedReference(token));
return IArrayDimension.UNBOUND_VALUE;
Expand Down Expand Up @@ -1335,9 +1298,9 @@ private int calculateLengthInBytesWithArray(ITypedVariableNode target)
return (target.type().byteSize()) * totalOccurrences;
}

private boolean isVariableDeclared(String potentialVariableName)
private boolean isVariableUndeclared(String potentialVariableName)
{
return declaredVariables.containsKey(potentialVariableName.toUpperCase());
return !declaredVariables.containsKey(potentialVariableName.toUpperCase());
}

private VariableNode getDeclaredVariable(ITokenNode tokenNode)
Expand Down Expand Up @@ -1401,9 +1364,10 @@ private void addVariableToCurrentGroup(VariableNode variable)
return;
}

if (groupStack.peekLast().level() == variable.level() - 1)
var last = groupStack.peekLast();
if (last.level() == variable.level() - 1)
{
groupStack.peekLast().addVariable(variable);
last.addVariable(variable);
}
}

Expand Down Expand Up @@ -1469,6 +1433,7 @@ private boolean hasMixedConst()
}
}

@SuppressWarnings("ClassEscapesDefinedScope")
@Override
protected ITokenNode consumeMandatoryIdentifierTokenNode(BaseSyntaxNode node)
{
Expand Down
49 changes: 35 additions & 14 deletions libs/natparse/src/test/java/org/amshove/natparse/parsing/DefineDataParserShould.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,21 @@ void allowMixedStringConcatInitialValuesForAlphanumericFields()
assertThat(stringConcat.stringValue()).isEqualTo("Hello\nWorld");
}

@Test
void allowMixedStringConcatInitialValuesStartingWithHex()
{
var defineData = assertParsesWithoutDiagnostics("""
define data local
1 alfa (a29) init<H'0A'
-'Test'>
end-define
""");

var variable = assertNodeType(defineData.variables().first(), ITypedVariableNode.class);
var concat = assertNodeType(variable.type().initialValue(), IStringConcatOperandNode.class);
assertThat(concat.stringValue()).isEqualTo("\nTest");
}

@Test
void allowNumericConstValuesForAlphanumericFields()
{
Expand Down Expand Up @@ -529,6 +544,7 @@ void parseGroupVariables()

var scopeNode = defineData.findDescendantOfType(IScopeNode.class);
assertThat(scopeNode).isNotNull();
assert scopeNode != null;
assertThat(scopeNode.descendants().size()).isEqualTo(3); // LOCAL + Group + Typed

var group = assertNodeType(defineData.variables().first(), IGroupNode.class);
Expand Down Expand Up @@ -675,6 +691,7 @@ void parseAnArrayWithWhitespaceBeforeTheSlash()
""");
}

@Test
void parseAnArrayWithWhitespaceAfterTheSlash()
{
assertParsesWithoutDiagnostics("""
Expand Down Expand Up @@ -907,6 +924,8 @@ void addAReferenceToTheConstantForConstArrayDimension()
var length = assertNodeType(defineData.variables().first(), IReferencableNode.class);
var myArray = assertNodeType(defineData.variables().last(), ITypedVariableNode.class);
var referenceNode = myArray.dimensions().first().findDescendantOfType(ISymbolReferenceNode.class);
assertThat(referenceNode).as("No reference found").isNotNull();
assert referenceNode != null;

assertThat(length.references().first()).isEqualTo(referenceNode);
assertThat(referenceNode.reference()).isEqualTo(length);
Expand Down Expand Up @@ -1021,6 +1040,7 @@ void parseTheUpperBoundOfVariableParameterDimensions()

var parameter = defineData.findVariable("#PARM");
assertThat(parameter).isNotNull();
assert parameter != null;
assertThat(parameter.dimensions().first().upperBound()).isEqualTo(10);
}

Expand All @@ -1036,6 +1056,7 @@ void parseTheUpperBoundOfVariableParameterDimensionsInGroups()

var parameter = defineData.findVariable("#PARM");
assertThat(parameter).isNotNull();
assert parameter != null;
assertThat(parameter.dimensions().first().upperBound()).isEqualTo(10);
}

Expand Down Expand Up @@ -1204,13 +1225,13 @@ void redefineIndependentVariables()
void notRaiseADiagnosticIfRedefineHasSmallerOrEqualLength(String varFormat, String redefVarFormat)
{
var source = """
DEFINE DATA
LOCAL
1 #FIELD (%s)
1 REDEFINE #FIELD
2 #REDEF-FIELD (%s)
END-DEFINE
""".formatted(varFormat, redefVarFormat);
DEFINE DATA
LOCAL
1 #FIELD (%s)
1 REDEFINE #FIELD
2 #REDEF-FIELD (%s)
END-DEFINE
""".formatted(varFormat, redefVarFormat);

assertParsesWithoutDiagnostics(source);
}
Expand All @@ -1224,13 +1245,13 @@ void notRaiseADiagnosticIfRedefineHasSmallerOrEqualLength(String varFormat, Stri
void raiseADiagnosticIfRedefineHasGreaterLength(String varFormat, String redefVarFormat)
{
var source = """
DEFINE DATA
LOCAL
1 #FIELD (%s)
1 REDEFINE #FIELD
2 #REDEF-FIELD (%s)
END-DEFINE
""".formatted(varFormat, redefVarFormat);
DEFINE DATA
LOCAL
1 #FIELD (%s)
1 REDEFINE #FIELD
2 #REDEF-FIELD (%s)
END-DEFINE
""".formatted(varFormat, redefVarFormat);

assertDiagnostic(source, ParserError.REDEFINE_LENGTH_EXCEEDS_TARGET_LENGTH);
}
Expand Down

0 comments on commit 6c9ca4f

Please sign in to comment.