Skip to content

Commit

Permalink
fixed: number formats in objectpascal lexer (#887)
Browse files Browse the repository at this point in the history 
* fixed: number formats in objectpascal lexer

added: some missing types
added: special compiler directives

* updated: testdata for objectpascal lexer
  • Loading branch information
@codiacdev
codiacdev authored Nov 19, 2023
1 parent b5afdee commit 1cda665
Show file tree
Hide file tree
Showing 3 changed files with 235 additions and 8 deletions.
10 changes: 7 additions & 3 deletions lexers/embedded/objectpascal.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<token type="Text"/>
</rule>
<!-- Compiler Directive -->
<rule pattern="\{[$].*?\}">
<rule pattern="\{[$].*?\}|\{[-](NOD|EXT|OBJ).*?\}">
<token type="CommentPreproc" />
</rule>
<!-- Comment Single -->
Expand All @@ -51,10 +51,10 @@
<token type="Operator" />
</rule>
<!-- Numbers -->
<rule pattern="[\$][0-9a-fA-F]*[xX][0-9a-fA-F]*|[\$][0-9a-fA-F]*|[0-9]+[xX][0-9a-fA-F]*|([0-9]+[0-9a-fA-F]+(?=[hH]))">
<rule pattern="[\$][0-9a-fA-F]*[xX][0-9a-fA-F]*|[\$][0-9a-fA-F]*|[\#][0-9a-fA-F]*|[0-9]+[xX][0-9a-fA-F]*|([0-9]+[0-9a-fA-F]+(?=[hH]))">
<token type="LiteralNumberHex" />
</rule>
<rule pattern="([0-9][0-9_]*\.([0-9][0-9_]*)?|\.[0-9][0-9_]*)([eE][+\-]?[0-9][0-9_]*)?[fFdD]?|[0-9][eE][+\-]?[0-9][0-9_]*[fFdD]?|[0-9]([eE][+\-]?[0-9][0-9_]*)?[fFdD]|0[xX]([0-9a-fA-F][0-9a-fA-F_]*\.?|([0-9a-fA-F][0-9a-fA-F_]*)?\.[0-9a-fA-F][0-9a-fA-F_]*)[pP][+\-]?[0-9][0-9_]*[fFdD]?">
<rule pattern="[0-9]+(\&#39;[0-9]+)*\.[0-9]+(\&#39;[0-9]+)*[eE][+-]?[0-9]+(\&#39;[0-9]+)*|[0-9]+(\&#39;[0-9]+)*\.[0-9]+(\&#39;[0-9]+)*|\d+[eE][+-]?[0-9]+">
<token type="LiteralNumberFloat" />
</rule>
<rule pattern="0|[1-9][0-9_]*?">
Expand Down Expand Up @@ -84,6 +84,10 @@
<rule pattern="\b(?!=\.)(?i:(PChar|PAnsiChar|PWideChar|PRawByteString|PUnicodeString|PString|PAnsiString|PShortString|PTextBuf|PWideString|PByte|PShortInt|PWord|PSmallInt|PCardinal|PLongWord|PFixedUInt|PLongint|PFixedInt|PUInt64|PInt64|PNativeUInt|PNativeInt|PByteArray|PCurrency|PDouble|PExtended|PSingle|PInteger|POleVariant|PVarRec|PVariant|PWordArray|PBoolean|PWordBool|PLongBool|PPointer))\b(?![&#60;\/(])">
<token type="KeywordType" />
</rule>
<!-- More Types -->
<rule pattern="\b(?!=\.)(?i:(IntPtr|UIntPtr|Float32|Float64|_ShortStr|_ShortString|_AnsiStr|_AnsiString|_AnsiChr|_AnsiChar|_WideStr|_WideString|_PAnsiChr|_PAnsiChar|UTF8Char|_AnsiChar|PUTF8Char|_PAnsiChar|MarshaledString|MarshaledAString))\b(?![&#60;\/(])">
<token type="KeywordType" />
</rule>
<!-- Result -->
<rule pattern="\b(?!=\.)(?i:(Result))\b(?![&#60;\/(])">
<token type="GenericEmph" />
Expand Down
27 changes: 26 additions & 1 deletion lexers/testdata/objectpascal.actual
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,38 @@ const
and now '''
some more text
''';

const
cMultiLineString2 = '''''
some text
and now '''
some more text
''''';

var
fp1: real := 6.123e-22;
fp2: real := 612.3e-24;
fp3: real := 61.23e-23;
fp4: real := 1.1e-8;
Epsilon: Single = 1E-40;

{$NODEFINE string 'UnicodeString' } {$OBJTYPENAME string 'NUnicodeString'} { defined in ustring.h }
{-NODEFINE string 'String' } {$OBJTYPENAME string 'NUnicodeString'} { defined in ustring.h }
{-EXTERNALSYM ShortInt 'signed char' } {-OBJTYPENAME ShortInt 'Bzc'}

const
CLineBreak = {$IFDEF POSIX} _AnsiStr(#10) {$ENDIF}
{$IFDEF MSWINDOWS} _AnsiStr(#13#10) {$ENDIF};

IntegerArray = array[0..$effffff] of Integer;
PIntegerArray = ^IntegerArray;

PointerArray = array [0..512*1024*1024 - 2] of Pointer;
PPointerArray = ^PointerArray;

TPCharArray = packed array[0..(High(Integer) div SizeOf(PChar))-1] of PChar;
PPCharArray = ^TPCharArray;

var
LNativeInt: NativeInt;
LNativeUInt: NativeUInt;
Expand Down
206 changes: 202 additions & 4 deletions lexers/testdata/objectpascal.expected
View file
Original file line number Diff line number Diff line change
Expand Up @@ -325,19 +325,217 @@
{"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"'''\n some text\n and now '''\n some more text\n '''"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n\n"},
{"type":"KeywordReserved","value":"const"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"cMultiLineString2"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"'''''\n some text\n and now '''\n some more text\n '''''"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n\n"},
{"type":"KeywordReserved","value":"var"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"fp1"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"real"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":":="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberFloat","value":"6.123e-22"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"fp2"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"real"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":":="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberFloat","value":"612.3e-24"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"fp3"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"real"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":":="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberFloat","value":"61.23e-23"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"fp4"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"real"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":":="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberFloat","value":"1.1e-8"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Epsilon"},
{"type":"Operator","value":":"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"Single"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberFloat","value":"1E-40"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n\n"},
{"type":"CommentPreproc","value":"{$NODEFINE string 'UnicodeString' }"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"{$OBJTYPENAME string 'NUnicodeString'}"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentMultiline","value":"{ defined in ustring.h }"},
{"type":"Text","value":"\n"},
{"type":"CommentPreproc","value":"{-NODEFINE string 'String' }"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"{$OBJTYPENAME string 'NUnicodeString'}"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentMultiline","value":"{ defined in ustring.h }"},
{"type":"Text","value":"\n"},
{"type":"CommentPreproc","value":"{-EXTERNALSYM ShortInt 'signed char' }"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"{-OBJTYPENAME ShortInt 'Bzc'}"},
{"type":"Text","value":"\n\n"},
{"type":"KeywordReserved","value":"const"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"CLineBreak"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"{$IFDEF POSIX}"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"_AnsiStr"},
{"type":"Operator","value":"("},
{"type":"LiteralNumberHex","value":"#10"},
{"type":"Operator","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"{$ENDIF}"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"{$IFDEF MSWINDOWS}"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"_AnsiStr"},
{"type":"Operator","value":"("},
{"type":"LiteralNumberHex","value":"#13#10"},
{"type":"Operator","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"CommentPreproc","value":"{$ENDIF}"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"cMultiLineString2"},
{"type":"Text","value":"IntegerArray"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"array"},
{"type":"Punctuation","value":"["},
{"type":"LiteralNumberInteger","value":"0"},
{"type":"Operator","value":".."},
{"type":"LiteralNumberHex","value":"$effffff"},
{"type":"Punctuation","value":"]"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"of"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"Integer"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"PIntegerArray"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"'''''\n some text\n and now '''\n some more text\n '''''"},
{"type":"Operator","value":"^"},
{"type":"Text","value":"IntegerArray"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"PointerArray"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"array"},
{"type":"TextWhitespace","value":" "},
{"type":"Punctuation","value":"["},
{"type":"LiteralNumberInteger","value":"0"},
{"type":"Operator","value":".."},
{"type":"LiteralNumberInteger","value":"512"},
{"type":"Operator","value":"*"},
{"type":"LiteralNumberInteger","value":"1024"},
{"type":"Operator","value":"*"},
{"type":"LiteralNumberInteger","value":"1024"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"-"},
{"type":"TextWhitespace","value":" "},
{"type":"LiteralNumberInteger","value":"2"},
{"type":"Punctuation","value":"]"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"of"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"Pointer"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"PPointerArray"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"^"},
{"type":"Text","value":"PointerArray"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"TPCharArray"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"packed"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"array"},
{"type":"Punctuation","value":"["},
{"type":"LiteralNumberInteger","value":"0"},
{"type":"Operator","value":"..("},
{"type":"Text","value":"High"},
{"type":"Operator","value":"("},
{"type":"KeywordType","value":"Integer"},
{"type":"Operator","value":")"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"div"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"SizeOf"},
{"type":"Operator","value":"("},
{"type":"KeywordType","value":"PChar"},
{"type":"Operator","value":"))-"},
{"type":"LiteralNumberInteger","value":"1"},
{"type":"Punctuation","value":"]"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordReserved","value":"of"},
{"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"PChar"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"PPCharArray"},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
{"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"^"},
{"type":"Text","value":"TPCharArray"},
{"type":"Punctuation","value":";"},
{"type":"Text","value":"\n\n"},
{"type":"KeywordReserved","value":"var"},
{"type":"Text","value":"\n"},
{"type":"TextWhitespace","value":" "},
Expand Down Expand Up @@ -1526,8 +1724,8 @@
{"type":"Operator","value":"\u003e"},
{"type":"TextWhitespace","value":" "},
{"type":"Text","value":"Ord"},
{"type":"Operator","value":"(#"},
{"type":"LiteralNumberInteger","value":"10"},
{"type":"Operator","value":"("},
{"type":"LiteralNumberHex","value":"#10"},
{"type":"Operator","value":")"},
{"type":"Punctuation","value":"."},
{"type":"Text","value":"ToExtended"},
Expand Down

0 comments on commit 1cda665

Please sign in to comment.