diff --git a/pom.xml b/pom.xml index a06aea264..781110c47 100644 --- a/pom.xml +++ b/pom.xml @@ -210,7 +210,7 @@ com.helger.maven ph-javacc-maven-plugin - 4.1.3 + 4.1.4-SNAPSHOT phase1 diff --git a/src/main/java/com/helger/pgcc/output/java/JavaResourceTemplateLocationImpl.java b/src/main/java/com/helger/pgcc/output/java/JavaResourceTemplateLocationImpl.java index c284251d4..e5b6fdb17 100644 --- a/src/main/java/com/helger/pgcc/output/java/JavaResourceTemplateLocationImpl.java +++ b/src/main/java/com/helger/pgcc/output/java/JavaResourceTemplateLocationImpl.java @@ -33,8 +33,14 @@ */ package com.helger.pgcc.output.java; +import javax.annotation.Nonnull; + +import com.helger.commons.annotation.Nonempty; + public class JavaResourceTemplateLocationImpl implements IJavaResourceTemplateLocations { + @Nonnull + @Nonempty public String getTokenTemplateResourceUrl () { return "/templates/Token.template"; diff --git a/src/main/resources/templates/stream/java/AbstractCharStream.template b/src/main/resources/templates/stream/java/AbstractCharStream.template index f31e16d0d..eb9a55adc 100644 --- a/src/main/resources/templates/stream/java/AbstractCharStream.template +++ b/src/main/resources/templates/stream/java/AbstractCharStream.template @@ -11,157 +11,213 @@ implements CharStream { /** Default buffer size if nothing is specified */ public static final int DEFAULT_BUF_SIZE = 4096; - /** By how much should the buffer be increase? */ - protected static final int BUFFER_INCREASE = 2048; - static final int hexval(final char c) throws java.io.IOException { - switch(c) + static final int hexval (final char c) throws java.io.IOException + { + switch (c) { - case '0' : - return 0; - case '1' : - return 1; - case '2' : - return 2; - case '3' : - return 3; - case '4' : - return 4; - case '5' : - return 5; - case '6' : - return 6; - case '7' : - return 7; - case '8' : - return 8; - case '9' : - return 9; - case 'a' : - case 'A' : - return 10; - case 'b' : - case 'B' : - return 11; - case 'c' : - case 'C' : - return 12; - case 'd' : - case 'D' : - return 13; - case 'e' : - case 'E' : - return 14; - case 'f' : - case 'F' : - return 15; + case '0': + return 0; + case '1': + return 1; + case '2': + return 2; + case '3': + return 3; + case '4': + return 4; + case '5': + return 5; + case '6': + return 6; + case '7': + return 7; + case '8': + return 8; + case '9': + return 9; + case 'a': + case 'A': + return 10; + case 'b': + case 'B': + return 11; + case 'c': + case 'C': + return 12; + case 'd': + case 'D': + return 13; + case 'e': + case 'E': + return 14; + case 'f': + case 'F': + return 15; + default: + throw new java.io.IOException ("Invalid hex char '" + c + "' (=" + (int) c + ") provided!"); } - - // Should never come here - throw new java.io.IOException("Invalid hex char '" + c + "' provided!"); } + /** Tab size for formatting. Usually in the range 1 to 8. */ + private int m_nTabSize = 1; + /** Position in buffer. */ - protected int bufpos = -1; + protected int bufpos; protected int bufsize; protected int available; protected int tokenBegin; + /** Internal backup buffer */ protected char[] buffer; - protected int inBuf = 0; - private int tabSize = 1; - protected int maxNextCharInd = 0; + /** Characters in the backup buffer */ + protected int inBuf; + protected int maxNextCharInd; #if KEEP_LINE_COLUMN - protected int[] bufline; - protected int[] bufcolumn; + private int[] m_aBufLine; + private int[] m_aBufColumn; + + // Current line number + private int m_nLineNo; + // Current column number + private int m_nColumnNo; + + // Was the previous character a "\r" char? + private boolean m_bPrevCharIsCR; + // Was the previous character a "\n" char? + private boolean m_bPrevCharIsLF; + + // Is line/column tracking enabled? + private boolean m_bTrackLineColumn = true; +#fi - protected int column = 0; - protected int line = 1; - protected boolean prevCharIsCR = false; - protected boolean prevCharIsLF = false; - private boolean trackLineColumn = true; -#fi + /** Constructor. */ + public AbstractCharStream(final int nStartLine, + final int nStartColumn, + final int nBufferSize) + { + reInit (nStartLine, nStartColumn, nBufferSize); + } - public void setTabSize(final int i) - { - tabSize = i; + /** Reinitialise. */ + public final void reInit(final int nStartLine, + final int nStartColumn, + final int nBufferSize) + { +#if KEEP_LINE_COLUMN + m_nLineNo = nStartLine; + m_nColumnNo = nStartColumn - 1; + m_bPrevCharIsCR = false; + m_bPrevCharIsLF = false; +#fi + if (buffer == null || nBufferSize != buffer.length) + { + bufsize = nBufferSize; + available = nBufferSize; + buffer = new char[nBufferSize]; +#if KEEP_LINE_COLUMN + m_aBufLine = new int[nBufferSize]; + m_aBufColumn = new int[nBufferSize]; +#fi + } + maxNextCharInd = 0; + inBuf = 0; + tokenBegin = 0; + bufpos = -1; } + + /** + * Read from the underlying stream. + * @param aBuf the buffer to be filled + * @param nOfs The offset into the buffer. 0-based + * @param nLen Number of chars to read. + * @return Number of effective chars read, or -1 on error. + */ + protected abstract int streamRead (char[] aBuf, int nOfs, int nLen) throws java.io.IOException; - public int getTabSize() - { - return tabSize; + /** + * Close the underlying stream. + * @throws java.io.IOException If closing fails. + */ + protected abstract void streamClose () throws java.io.IOException; + + // Override this method if you need more aggressive buffer size expansion + protected int getBufSizeAfterExpansion () + { + // Double the size by default + return bufsize * 2; } - protected void expandBuff(final boolean wrapAround) + protected void expandBuff (final boolean bWrapAround) { - final char[] newbuffer = new char[bufsize + BUFFER_INCREASE]; + // Get the new buffer size + final int nNewBufSize = getBufSizeAfterExpansion (); + + final char[] newbuffer = new char[nNewBufSize]; #if KEEP_LINE_COLUMN - final int newbufline[] = new int[bufsize + BUFFER_INCREASE]; - final int newbufcolumn[] = new int[bufsize + BUFFER_INCREASE]; + final int[] newbufline = new int[nNewBufSize]; + final int[] newbufcolumn = new int[nNewBufSize]; #fi - try + // Number of chars to be preserved + final int nPreservedChars = bufsize - tokenBegin; + + if (bWrapAround) { - if (wrapAround) - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - System.arraycopy(buffer, 0, newbuffer, bufsize - tokenBegin, bufpos); - buffer = newbuffer; + // From offset "tokenBegin" to offset 0 + + // src, srcPos, dest, destPos, length + System.arraycopy(buffer, tokenBegin, newbuffer, 0, nPreservedChars); + System.arraycopy(buffer, 0, newbuffer, nPreservedChars, bufpos); + buffer = newbuffer; #if KEEP_LINE_COLUMN - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos); - bufline = newbufline; + System.arraycopy(m_aBufLine, tokenBegin, newbufline, 0, nPreservedChars); + System.arraycopy(m_aBufLine, 0, newbufline, nPreservedChars, bufpos); + m_aBufLine = newbufline; - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos); - bufcolumn = newbufcolumn; + System.arraycopy(m_aBufColumn, tokenBegin, newbufcolumn, 0, nPreservedChars); + System.arraycopy(m_aBufColumn, 0, newbufcolumn, nPreservedChars, bufpos); + m_aBufColumn = newbufcolumn; #fi - bufpos += (bufsize - tokenBegin); - // https://github.com/phax/ParserGeneratorCC/pull/23 - maxNextCharInd = bufpos; - } - else - { - System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin); - buffer = newbuffer; + bufpos += nPreservedChars; + maxNextCharInd = bufpos; + } + else + { + // From offset "tokenBegin" to offset 0 + + System.arraycopy(buffer, tokenBegin, newbuffer, 0, nPreservedChars); + buffer = newbuffer; #if KEEP_LINE_COLUMN - System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin); - bufline = newbufline; + System.arraycopy(m_aBufLine, tokenBegin, newbufline, 0, nPreservedChars); + m_aBufLine = newbufline; - System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin); - bufcolumn = newbufcolumn; + System.arraycopy(m_aBufColumn, tokenBegin, newbufcolumn, 0, nPreservedChars); + m_aBufColumn = newbufcolumn; #fi - bufpos -= tokenBegin; - // https://github.com/phax/ParserGeneratorCC/pull/23 - maxNextCharInd = bufpos; - } - } - catch (final Exception ex) - { - throw new IllegalStateException(ex); + bufpos -= tokenBegin; + maxNextCharInd = bufpos; } - bufsize += BUFFER_INCREASE; - available = bufsize; + // Increase buffer size + bufsize = nNewBufSize; + available = nNewBufSize; tokenBegin = 0; } - protected abstract int streamRead(char[] aBuf, int nOfs, int nLen) throws java.io.IOException; - - protected abstract void streamClose() throws java.io.IOException; - protected void fillBuff() throws java.io.IOException { if (maxNextCharInd == available) { if (available == bufsize) { + // ?What is the reason for this? if (tokenBegin > 2048) { bufpos = 0; @@ -189,87 +245,99 @@ implements CharStream try { - final int i = streamRead(buffer, maxNextCharInd, available - maxNextCharInd); - if (i == -1) + // Read from underlying stream + final int nCharsRead = streamRead (buffer, maxNextCharInd, available - maxNextCharInd); + if (nCharsRead == -1) { - streamClose(); - throw new java.io.IOException(); + // We reached the end of the file + streamClose (); + + // Caught down below and re-thrown + throw new java.io.IOException("PGCC end of stream"); } - maxNextCharInd += i; + maxNextCharInd += nCharsRead; } catch (final java.io.IOException ex) { --bufpos; - backup(0); + // ?What is the reason of this? + backup (0); if (tokenBegin == -1) + { + // Error occurred in beginToken() tokenBegin = bufpos; + } throw ex; } } - - public char beginToken() throws java.io.IOException +#if KEEP_LINE_COLUMN + + protected final void internalSetBufLineColumn (final int nLine, final int nColumn) { - tokenBegin = -1; - final char c = readChar(); - tokenBegin = bufpos; - return c; + m_aBufLine[bufpos] = nLine; + m_aBufColumn[bufpos] = nColumn; } -#if KEEP_LINE_COLUMN - protected void updateLineColumn(char c) + protected final void internalUpdateLineColumn(final char c) { - column++; + m_nColumnNo++; - if (prevCharIsLF) + if (m_bPrevCharIsLF) { - prevCharIsLF = false; - column = 1; - line++; + // It's a "\r\n" or "\n" + // Start of a new line + m_bPrevCharIsLF = false; + m_nColumnNo = 1; + m_nLineNo++; } else - if (prevCharIsCR) + if (m_bPrevCharIsCR) { - prevCharIsCR = false; + m_bPrevCharIsCR = false; if (c == '\n') - prevCharIsLF = true; + { + // It's a "\r\n" + m_bPrevCharIsLF = true; + } else { - column = 1; - line++; + // It's only a "\r" + m_nColumnNo = 1; + m_nLineNo++; } } switch (c) { - case '\r' : - prevCharIsCR = true; - break; - case '\n' : - prevCharIsLF = true; + case '\r': + m_bPrevCharIsCR = true; break; - case '\t' : - column--; - column += (tabSize - (column % tabSize)); + case '\n': + m_bPrevCharIsLF = true; break; - default : + case '\t': + m_nColumnNo--; + m_nColumnNo += (m_nTabSize - (m_nColumnNo % m_nTabSize)); break; } - bufline[bufpos] = line; - bufcolumn[bufpos] = column; + internalSetBufLineColumn (m_nLineNo, m_nColumnNo); } #fi - /** Read a character. */ public char readChar() throws java.io.IOException { if (inBuf > 0) { + // Something is left from last backup --inBuf; ++bufpos; if (bufpos == bufsize) + { + // Buffer overflow bufpos = 0; + } return buffer[bufpos]; } @@ -278,98 +346,69 @@ implements CharStream if (bufpos >= maxNextCharInd) fillBuff(); - char c = buffer[bufpos]; + final char c = buffer[bufpos]; #if KEEP_LINE_COLUMN - if (trackLineColumn) - updateLineColumn(c); + if (m_bTrackLineColumn) + internalUpdateLineColumn(c); #fi return c; } - public int getBeginColumn() { -#if KEEP_LINE_COLUMN - return bufcolumn[tokenBegin]; -#else - return -1; -#fi + public char beginToken() throws java.io.IOException + { + tokenBegin = -1; + final char c = readChar(); + tokenBegin = bufpos; + return c; } - public int getBeginLine() { + public int getBeginColumn () + { #if KEEP_LINE_COLUMN - return bufline[tokenBegin]; + return m_aBufColumn[tokenBegin]; #else return -1; #fi } - public int getEndColumn() { + public int getBeginLine () + { #if KEEP_LINE_COLUMN - return bufcolumn[bufpos]; + return m_aBufLine[tokenBegin]; #else return -1; #fi } - public int getEndLine() { + public int getEndColumn () + { #if KEEP_LINE_COLUMN - return bufline[bufpos]; + return m_aBufColumn[bufpos]; #else return -1; #fi } - public void backup(final int amount) { - inBuf += amount; - bufpos -= amount; - if (bufpos < 0) - bufpos += bufsize; - } - - /** Constructor. */ - public AbstractCharStream(final int startline, - final int startcolumn, - final int buffersize) + public int getEndLine () { #if KEEP_LINE_COLUMN - line = startline; - column = startcolumn - 1; -#fi - - bufsize = buffersize; - available = buffersize; - buffer = new char[buffersize]; -#if KEEP_LINE_COLUMN - bufline = new int[buffersize]; - bufcolumn = new int[buffersize]; + return m_aBufLine[bufpos]; +#else + return -1; #fi } - /** Reinitialise. */ - public void reInit(final int startline, - final int startcolumn, - final int buffersize) + // Backup amount characters + public void backup (final int nAmount) { -#if KEEP_LINE_COLUMN - line = startline; - column = startcolumn - 1; - prevCharIsCR = false; - prevCharIsLF = false; -#fi - if (buffer == null || buffersize != buffer.length) + inBuf += nAmount; + bufpos -= nAmount; + if (bufpos < 0) { - bufsize = buffersize; - available = buffersize; - buffer = new char[buffersize]; -#if KEEP_LINE_COLUMN - bufline = new int[buffersize]; - bufcolumn = new int[buffersize]; -#fi + // Buffer underflow (modulo) + bufpos += bufsize; } - maxNextCharInd = 0; - inBuf = 0; - tokenBegin = 0; - bufpos = -1; } public String getImage() @@ -377,23 +416,28 @@ implements CharStream if (bufpos >= tokenBegin) { // from tokenBegin to bufpos - return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); + return new String (buffer, tokenBegin, bufpos - tokenBegin + 1); } - // from tokenBegin to bufpos including wrap around - return new String(buffer, tokenBegin, bufsize - tokenBegin) + - new String(buffer, 0, bufpos + 1); + // from tokenBegin to bufsize, and from 0 to bufpos + return new String (buffer, tokenBegin, bufsize - tokenBegin) + + new String (buffer, 0, bufpos + 1); } - public char[] getSuffix(final int len) + public char[] getSuffix (final int len) { char[] ret = new char[len]; if ((bufpos + 1) >= len) + { + // one piece System.arraycopy(buffer, bufpos - len + 1, ret, 0, len); + } else { - System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0, len - bufpos - 1); - System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1); + // Wrap around + final int nPart1 = len - bufpos - 1; + System.arraycopy(buffer, bufsize - nPart1, ret, 0, nPart1); + System.arraycopy(buffer, 0, ret, nPart1, bufpos + 1); } return ret; } @@ -402,21 +446,32 @@ implements CharStream { buffer = null; #if KEEP_LINE_COLUMN - bufline = null; - bufcolumn = null; + m_aBufLine = null; + m_aBufColumn = null; #fi } + + public final int getTabSize() + { + return m_nTabSize; + } + + public final void setTabSize (final int nTabSize) + { + m_nTabSize = nTabSize; + } #if KEEP_LINE_COLUMN /** * Method to adjust line and column numbers for the start of a token. + * This is used internally to */ - public void adjustBeginLineColumn(final int nNewLine, final int newCol) + public final void adjustBeginLineColumn(final int nNewLine, final int newCol) { int start = tokenBegin; int newLine = nNewLine; + int len; - if (bufpos >= tokenBegin) { len = bufpos - tokenBegin + inBuf + 1; @@ -433,40 +488,58 @@ implements CharStream int columnDiff = 0; // TODO disassemble meaning and split up - while (i < len && bufline[j = start % bufsize] == bufline[k = ++start % bufsize]) + while (i < len && m_aBufLine[j = start % bufsize] == m_aBufLine[k = ++start % bufsize]) { - bufline[j] = newLine; - nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j]; - bufcolumn[j] = newCol + columnDiff; + m_aBufLine[j] = newLine; + nextColDiff = columnDiff + m_aBufColumn[k] - m_aBufColumn[j]; + m_aBufColumn[j] = newCol + columnDiff; columnDiff = nextColDiff; i++; } if (i < len) { - bufline[j] = newLine++; - bufcolumn[j] = newCol + columnDiff; + m_aBufLine[j] = newLine++; + m_aBufColumn[j] = newCol + columnDiff; while (i++ < len) { // TODO disassemble meaning and split up - if (bufline[j = start % bufsize] != bufline[++start % bufsize]) - bufline[j] = newLine++; + if (m_aBufLine[j = start % bufsize] != m_aBufLine[++start % bufsize]) + m_aBufLine[j] = newLine++; else - bufline[j] = newLine; + m_aBufLine[j] = newLine; } } - line = bufline[j]; - column = bufcolumn[j]; + m_nLineNo = m_aBufLine[j]; + m_nColumnNo = m_aBufColumn[j]; } - - public void setTrackLineColumn(final boolean tlc) { - trackLineColumn = tlc; + + /** + * @return the current line number. 0-based. + */ + protected final int getLine () + { + return m_nLineNo; + } + + /** + * @return the current column number. 0-based. + */ + protected final int getColumn () + { + return m_nColumnNo; } - public boolean isTrackLineColumn() { - return trackLineColumn; + public final boolean isTrackLineColumn () + { + return m_bTrackLineColumn; + } + + public final void setTrackLineColumn (final boolean bTrackLineColumn) + { + m_bTrackLineColumn = bTrackLineColumn; } #fi } diff --git a/src/main/resources/templates/stream/java/CharStream.template b/src/main/resources/templates/stream/java/CharStream.template index c3d2d7f59..155e40f50 100644 --- a/src/main/resources/templates/stream/java/CharStream.template +++ b/src/main/resources/templates/stream/java/CharStream.template @@ -95,28 +95,30 @@ interface CharStream { * affect the lexer's operation. */ void done(); + + // Getters and setters + + /** + * @return Current tab size. + */ + int getTabSize(); /** * Set the tab size to use. * @param i spaces per tab */ void setTabSize(int i); - +#if KEEP_LINE_COLUMN + /** - * @return Current tab size. + * @return true if line number and column numbers should be tracked. */ - int getTabSize(); -#if KEEP_LINE_COLUMN + boolean isTrackLineColumn(); /** * Enable or disable line number and column number tracking. * @param trackLineColumn true to track it, false to not do it. */ void setTrackLineColumn(boolean trackLineColumn); - - /** - * @return true if line number and column numbers should be tracked. - */ - boolean isTrackLineColumn(); #fi } diff --git a/src/main/resources/templates/stream/java/JavaCharStream.template b/src/main/resources/templates/stream/java/JavaCharStream.template index 252acb296..945064fc8 100644 --- a/src/main/resources/templates/stream/java/JavaCharStream.template +++ b/src/main/resources/templates/stream/java/JavaCharStream.template @@ -10,39 +10,42 @@ class JavaCharStream extends AbstractCharStream /** Predefined buffer size */ protected static final int NEXTCHAR_BUF_SIZE = 4096; - protected char[] nextCharBuf; - protected int nextCharInd = -1; - protected java.io.Reader inputStream; + private char[] m_aNextCharBuf; + private int nextCharInd = -1; + private java.io.Reader m_aIS; @Override protected int streamRead(final char[] buffer, final int offset, final int len) throws java.io.IOException { - return inputStream.read (buffer, offset, len); + return m_aIS.read (buffer, offset, len); } @Override protected void streamClose() throws java.io.IOException { - inputStream.close (); + m_aIS.close (); } @Override protected void fillBuff() throws java.io.IOException { if (maxNextCharInd == NEXTCHAR_BUF_SIZE) - maxNextCharInd = nextCharInd = 0; + { + maxNextCharInd = 0; + nextCharInd = 0; + } try { - final int i = inputStream.read(nextCharBuf, maxNextCharInd, NEXTCHAR_BUF_SIZE - maxNextCharInd); - if (i == -1) + final int nCharsRead = streamRead (m_aNextCharBuf, maxNextCharInd, NEXTCHAR_BUF_SIZE - maxNextCharInd); + if (nCharsRead == -1) { - inputStream.close(); - throw new java.io.IOException(); + streamClose (); + throw new java.io.IOException (); } - maxNextCharInd += i; + maxNextCharInd += nCharsRead; } - catch(final java.io.IOException ex) + catch (final java.io.IOException ex) { if (bufpos != 0) { @@ -51,25 +54,22 @@ class JavaCharStream extends AbstractCharStream } #if KEEP_LINE_COLUMN else - { - bufline[bufpos] = line; - bufcolumn[bufpos] = column; - } + internalSetBufLineColumn (getLine (), getColumn ()); #fi throw ex; } } - protected char readByte() throws java.io.IOException + private char readByte() throws java.io.IOException { - nextCharInd++; + ++nextCharInd; if (nextCharInd >= maxNextCharInd) fillBuff(); - return nextCharBuf[nextCharInd]; + return m_aNextCharBuf[nextCharInd]; } - /** @return starting character for token. */ + @Override public char beginToken() throws java.io.IOException { if (inBuf > 0) @@ -89,7 +89,7 @@ class JavaCharStream extends AbstractCharStream return readChar(); } - protected void adjustBuffSize() + private final void adjustBuffSize() { if (available == bufsize) { @@ -111,7 +111,7 @@ class JavaCharStream extends AbstractCharStream available = tokenBegin; } - /** Read a character. */ + @Override public char readChar() throws java.io.IOException { if (inBuf > 0) @@ -119,7 +119,10 @@ class JavaCharStream extends AbstractCharStream --inBuf; ++bufpos; if (bufpos == bufsize) + { + // Buffer overflow bufpos = 0; + } return buffer[bufpos]; } @@ -130,102 +133,126 @@ class JavaCharStream extends AbstractCharStream char c = readByte(); buffer[bufpos] = c; - if (c == '\\') + if (c != '\\') { + // Not a backslash #if KEEP_LINE_COLUMN if (isTrackLineColumn()) - updateLineColumn(c); + internalUpdateLineColumn(c); +#fi + return c; + } + +#if KEEP_LINE_COLUMN + if (isTrackLineColumn()) + internalUpdateLineColumn(c); #fi - int backSlashCnt = 1; + int backSlashCnt = 1; - for (;;) // Read all the backslashes - { - ++bufpos; - if (bufpos == available) - adjustBuffSize(); + // Read all the backslashes + for (;;) + { + ++bufpos; + if (bufpos == available) + adjustBuffSize(); - try + try + { + c = readByte(); + buffer[bufpos] = c; + if (c != '\\') { - c = readByte(); - buffer[bufpos] = c; - if (c != '\\') - { + // found a non-backslash char. #if KEEP_LINE_COLUMN - if (isTrackLineColumn()) - updateLineColumn(c); - + if (isTrackLineColumn()) + internalUpdateLineColumn(c); + #fi - // found a non-backslash char. - if ((c == 'u') && ((backSlashCnt & 1) == 1)) + if (c == 'u' && (backSlashCnt & 1) == 1) + { + --bufpos; + if (bufpos < 0) { - if (--bufpos < 0) - bufpos = bufsize - 1; - - break; + // Buffer underflow + bufpos = bufsize - 1; } - backup(backSlashCnt); - return '\\'; + // Found the start of a Unicode sequence + break; } - } - catch(final java.io.IOException e) - { - // We are returning one backslash so we should only backup (count-1) - if (backSlashCnt > 1) - backup(backSlashCnt-1); + // Unsupported escape sequence + backup(backSlashCnt); return '\\'; } + } + catch (final java.io.IOException e) + { + // We are returning one backslash so we should only backup (count-1) + if (backSlashCnt > 1) + backup (backSlashCnt - 1); + + return '\\'; + } + + // We read another backslash #if KEEP_LINE_COLUMN - if (isTrackLineColumn()) - updateLineColumn(c); + if (isTrackLineColumn()) + internalUpdateLineColumn (c); #fi - backSlashCnt++; - } + backSlashCnt++; + } - // Here, we have seen an odd number of backslash's followed by a 'u' - try + // Here, we have seen an odd number of backslash's followed by a 'u' + try + { + // Skip all 'u' as in '\uuuuuu1234' + while ((c = readByte()) == 'u') { - while ((c = readByte()) == 'u') #if KEEP_LINE_COLUMN - ++column; + if (isTrackLineColumn()) + internalUpdateLineColumn(c); #else - { /* empty */ } + /* empty */ #fi - - buffer[bufpos] = c = (char)(hexval(c) << 12 | - hexval(readByte()) << 8 | - hexval(readByte()) << 4 | - hexval(readByte())); + } + + final char c1 = c; + final char c2 = readByte(); + final char c3 = readByte(); + final char c4 = readByte(); + c = (char) (hexval(c1) << 12 | + hexval(c2) << 8 | + hexval(c3) << 4 | + hexval(c4)); + buffer[bufpos] = c; #if KEEP_LINE_COLUMN - column += 4; -#fi - } - catch(final java.io.IOException e) + if (isTrackLineColumn()) { + internalUpdateLineColumn(c1); + internalUpdateLineColumn(c2); + internalUpdateLineColumn(c3); + internalUpdateLineColumn(c4); + } +#fi + } + catch (final java.io.IOException ex) + { #if KEEP_LINE_COLUMN - throw new IllegalStateException("Invalid escape character at line " + line + " column " + column + "."); + throw new IllegalStateException("Invalid escape character at line " + getLine () + " column " + getColumn () + "."); #else - throw new IllegalStateException("Invalid escape character in input"); + throw new IllegalStateException("Invalid escape character in input"); #fi - } - - if (backSlashCnt == 1) - return c; - - backup(backSlashCnt - 1); - return '\\'; } - // Not a backslash -#if KEEP_LINE_COLUMN - if (isTrackLineColumn()) - updateLineColumn(c); -#fi - return c; + if (backSlashCnt == 1) + return c; + + backup(backSlashCnt - 1); + return '\\'; } /** Constructor. */ @@ -235,8 +262,8 @@ class JavaCharStream extends AbstractCharStream final int buffersize) { super (startline, startcolumn, buffersize); - nextCharBuf = new char[NEXTCHAR_BUF_SIZE]; - inputStream = dstream; + m_aNextCharBuf = new char[NEXTCHAR_BUF_SIZE]; + m_aIS = dstream; } /** Constructor. */ @@ -273,9 +300,9 @@ class JavaCharStream extends AbstractCharStream final int startcolumn, final int buffersize) { - nextCharBuf = new char[NEXTCHAR_BUF_SIZE]; + m_aNextCharBuf = new char[NEXTCHAR_BUF_SIZE]; nextCharInd = -1; - inputStream = dstream; + m_aIS = dstream; super.reInit (startline, startcolumn, buffersize); } @@ -374,7 +401,7 @@ class JavaCharStream extends AbstractCharStream @Override public void done () { - nextCharBuf = null; + m_aNextCharBuf = null; super.done (); } } diff --git a/src/main/resources/templates/stream/java/SimpleCharStream.template b/src/main/resources/templates/stream/java/SimpleCharStream.template index ae47f1aee..2a89cbf01 100644 --- a/src/main/resources/templates/stream/java/SimpleCharStream.template +++ b/src/main/resources/templates/stream/java/SimpleCharStream.template @@ -7,19 +7,7 @@ public #fi class SimpleCharStream extends AbstractCharStream { - protected java.io.Reader inputStream; - - @Override - protected int streamRead (final char[] aBuf, final int nOfs, final int nLen) throws java.io.IOException - { - return inputStream.read (aBuf, nOfs, nLen); - } - - @Override - protected void streamClose () throws java.io.IOException - { - inputStream.close (); - } + private java.io.Reader m_aIS; /** Constructor. */ public SimpleCharStream(final java.io.Reader dstream, @@ -28,7 +16,7 @@ class SimpleCharStream extends AbstractCharStream final int buffersize) { super (startline, startcolumn, buffersize); - inputStream = dstream; + m_aIS = dstream; } /** Constructor. */ @@ -51,7 +39,7 @@ class SimpleCharStream extends AbstractCharStream final int startcolumn, final int buffersize) { - inputStream = dstream; + m_aIS = dstream; super.reInit (startline, startcolumn, buffersize); } @@ -160,4 +148,17 @@ class SimpleCharStream extends AbstractCharStream { reInit(new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize); } + + @Override + protected int streamRead (final char[] aBuf, final int nOfs, final int nLen) throws java.io.IOException + { + return m_aIS.read (aBuf, nOfs, nLen); + } + + @Override + protected void streamClose () throws java.io.IOException + { + if (m_aIS != null) + m_aIS.close (); + } } diff --git a/src/main/resources/templates/stream/java/modern/JavaCharStream.template b/src/main/resources/templates/stream/java/modern/JavaCharStream.template index 9669dcdc3..50ee580c2 100644 --- a/src/main/resources/templates/stream/java/modern/JavaCharStream.template +++ b/src/main/resources/templates/stream/java/modern/JavaCharStream.template @@ -10,36 +10,40 @@ class JavaCharStream extends AbstractCharStream /** Predefined buffer size */ protected static final int NEXTCHAR_BUF_SIZE = 4096; - protected char[] nextCharBuf; - protected int nextCharInd = -1; - protected Provider inputStream; + private char[] m_aNextCharBuf; + private int nextCharInd = -1; + private Provider m_aIS; @Override protected int streamRead(final char[] buffer, final int offset, final int len) throws java.io.IOException { - return inputStream.read (buffer, offset, len); + return m_aIS.read (buffer, offset, len); } @Override protected void streamClose() throws java.io.IOException { - inputStream.close (); + m_aIS.close (); } + @Override protected void fillBuff() throws java.io.IOException { if (maxNextCharInd == NEXTCHAR_BUF_SIZE) - maxNextCharInd = nextCharInd = 0; + { + maxNextCharInd = 0; + nextCharInd = 0; + } try { - final int i = inputStream.read(nextCharBuf, maxNextCharInd, NEXTCHAR_BUF_SIZE - maxNextCharInd); - if (i == -1) + final int nCharsRead = streamRead (m_aNextCharBuf, maxNextCharInd, NEXTCHAR_BUF_SIZE - maxNextCharInd); + if (nCharsRead == -1) { - inputStream.close(); - throw new java.io.IOException(); + streamClose (); + throw new java.io.IOException (); } - maxNextCharInd += i; + maxNextCharInd += nCharsRead; } catch(final java.io.IOException ex) { @@ -50,25 +54,22 @@ class JavaCharStream extends AbstractCharStream } #if KEEP_LINE_COLUMN else - { - bufline[bufpos] = line; - bufcolumn[bufpos] = column; - } + internalSetBufLineColumn (getLine (), getColumn ()); #fi throw ex; } } - protected char readByte() throws java.io.IOException + private char readByte() throws java.io.IOException { ++nextCharInd; if (nextCharInd >= maxNextCharInd) fillBuff(); - return nextCharBuf[nextCharInd]; + return m_aNextCharBuf[nextCharInd]; } - /** @return starting character for token. */ + @Override public char beginToken() throws java.io.IOException { if (inBuf > 0) @@ -88,7 +89,7 @@ class JavaCharStream extends AbstractCharStream return readChar(); } - protected void adjustBuffSize() + private final void adjustBuffSize() { if (available == bufsize) { @@ -110,15 +111,18 @@ class JavaCharStream extends AbstractCharStream available = tokenBegin; } -/** Read a character. */ + @Override public char readChar() throws java.io.IOException { if (inBuf > 0) { --inBuf; - - if (++bufpos == bufsize) + ++bufpos; + if (bufpos == bufsize) + { + // Buffer overflow bufpos = 0; + } return buffer[bufpos]; } @@ -129,102 +133,126 @@ class JavaCharStream extends AbstractCharStream char c = readByte(); buffer[bufpos] = c; - if (c == '\\') + if (c != '\\') { + // Not a backslash #if KEEP_LINE_COLUMN if (isTrackLineColumn()) - updateLineColumn(c); + internalUpdateLineColumn(c); +#fi + return c; + } + +#if KEEP_LINE_COLUMN + if (isTrackLineColumn()) + internalUpdateLineColumn(c); #fi - int backSlashCnt = 1; + int backSlashCnt = 1; - for (;;) // Read all the backslashes - { - ++bufpos; - if (bufpos == available) - adjustBuffSize(); + // Read all the backslashes + for (;;) + { + ++bufpos; + if (bufpos == available) + adjustBuffSize(); - try + try + { + c = readByte(); + buffer[bufpos] = c; + if (c != '\\') { - c = readByte(); - buffer[bufpos] = c; - if (c != '\\') - { + // found a non-backslash char. #if KEEP_LINE_COLUMN - if (isTrackLineColumn()) - updateLineColumn(c); - + if (isTrackLineColumn()) + internalUpdateLineColumn(c); + #fi - // found a non-backslash char. - if ((c == 'u') && ((backSlashCnt & 1) == 1)) + if (c == 'u' && (backSlashCnt & 1) == 1) + { + --bufpos; + if (bufpos < 0) { - if (--bufpos < 0) - bufpos = bufsize - 1; - - break; + // Buffer underflow + bufpos = bufsize - 1; } - backup(backSlashCnt); - return '\\'; + // Found the start of a Unicode sequence + break; } - } - catch(final java.io.IOException e) - { - // We are returning one backslash so we should only backup (count-1) - if (backSlashCnt > 1) - backup(backSlashCnt-1); + // Unsupported escape sequence + backup(backSlashCnt); return '\\'; } + } + catch (final java.io.IOException e) + { + // We are returning one backslash so we should only backup (count-1) + if (backSlashCnt > 1) + backup (backSlashCnt - 1); + + return '\\'; + } + + // We read another backslash #if KEEP_LINE_COLUMN - if (isTrackLineColumn()) - updateLineColumn(c); + if (isTrackLineColumn()) + internalUpdateLineColumn (c); #fi - backSlashCnt++; - } + backSlashCnt++; + } - // Here, we have seen an odd number of backslash's followed by a 'u' - try + // Here, we have seen an odd number of backslash's followed by a 'u' + try + { + // Skip all 'u' as in '\uuuuuu1234' + while ((c = readByte()) == 'u') { - while ((c = readByte()) == 'u') #if KEEP_LINE_COLUMN - ++column; + if (isTrackLineColumn()) + internalUpdateLineColumn(c); #else - { /* empty */ } + /* empty */ #fi - - buffer[bufpos] = c = (char)(hexval(c) << 12 | - hexval(readByte()) << 8 | - hexval(readByte()) << 4 | - hexval(readByte())); + } + + final char c1 = c; + final char c2 = readByte(); + final char c3 = readByte(); + final char c4 = readByte(); + c = (char) (hexval(c1) << 12 | + hexval(c2) << 8 | + hexval(c3) << 4 | + hexval(c4)); + buffer[bufpos] = c; #if KEEP_LINE_COLUMN - column += 4; -#fi - } - catch(final java.io.IOException ex) + if (isTrackLineColumn()) { + internalUpdateLineColumn(c1); + internalUpdateLineColumn(c2); + internalUpdateLineColumn(c3); + internalUpdateLineColumn(c4); + } +#fi + } + catch (final java.io.IOException ex) + { #if KEEP_LINE_COLUMN - throw new IllegalStateException("Invalid escape character at line " + line + " column " + column + "."); + throw new IllegalStateException("Invalid escape character at line " + getLine () + " column " + getColumn () + "."); #else - throw new IllegalStateException("Invalid escape character in input"); + throw new IllegalStateException("Invalid escape character in input"); #fi - } - - if (backSlashCnt == 1) - return c; - - backup(backSlashCnt - 1); - return '\\'; } - // Not a backslash -#if KEEP_LINE_COLUMN - if (isTrackLineColumn()) - updateLineColumn(c); -#fi - return c; + if (backSlashCnt == 1) + return c; + + backup(backSlashCnt - 1); + return '\\'; } /** Constructor. */ @@ -234,8 +262,8 @@ class JavaCharStream extends AbstractCharStream final int buffersize) { super (startline, startcolumn, buffersize); - nextCharBuf = new char[NEXTCHAR_BUF_SIZE]; - inputStream = dstream; + m_aNextCharBuf = new char[NEXTCHAR_BUF_SIZE]; + m_aIS = dstream; } /** Constructor. */ @@ -272,16 +300,16 @@ class JavaCharStream extends AbstractCharStream final int startcolumn, final int buffersize) { - nextCharBuf = new char[NEXTCHAR_BUF_SIZE]; + m_aNextCharBuf = new char[NEXTCHAR_BUF_SIZE]; nextCharInd = -1; - inputStream = dstream; + m_aIS = dstream; super.reInit (startline, startcolumn, buffersize); } @Override public void done () { - nextCharBuf = null; + m_aNextCharBuf = null; super.done (); } } diff --git a/src/main/resources/templates/stream/java/modern/Provider.template b/src/main/resources/templates/stream/java/modern/Provider.template index 918de0f89..48956cb3f 100644 --- a/src/main/resources/templates/stream/java/modern/Provider.template +++ b/src/main/resources/templates/stream/java/modern/Provider.template @@ -18,7 +18,7 @@ extends java.io.Closeable * @return The number of characters read, or -1 if all read * @exception IOException if reading fails */ - int read(char[] aDest, int nOfs, int nLen) throws IOException; + int read (char [] aDest, int nOfs, int nLen) throws IOException; #if AT_LEAST_JDK7 #else diff --git a/src/main/resources/templates/stream/java/modern/SimpleCharStream.template b/src/main/resources/templates/stream/java/modern/SimpleCharStream.template index 7f3514c78..377abd952 100644 --- a/src/main/resources/templates/stream/java/modern/SimpleCharStream.template +++ b/src/main/resources/templates/stream/java/modern/SimpleCharStream.template @@ -7,7 +7,7 @@ public #fi class SimpleCharStream extends AbstractCharStream { - protected Provider inputStream; + private Provider inputStream; @Override protected int streamRead (final char[] aBuf, final int nOfs, final int nLen) throws java.io.IOException diff --git a/src/main/resources/templates/stream/java/modern/StreamProvider.template b/src/main/resources/templates/stream/java/modern/StreamProvider.template index abd3adaf7..2e4ec7207 100644 --- a/src/main/resources/templates/stream/java/modern/StreamProvider.template +++ b/src/main/resources/templates/stream/java/modern/StreamProvider.template @@ -8,25 +8,30 @@ import java.io.Reader; /** * NOTE : This generated class can be safely deleted if installing in a GWT installation (use StringProvider instead) */ -public class StreamProvider implements Provider { +public class StreamProvider implements Provider +{ private Reader m_aReader; #if AT_LEAST_JDK6 @Deprecated #fi - public StreamProvider(final InputStream stream, final String charsetName) throws IOException { - this (new BufferedReader(new InputStreamReader(stream, charsetName))); + public StreamProvider(final InputStream stream, final String charsetName) throws IOException + { + this (new BufferedReader (new InputStreamReader (stream, charsetName))); } - public StreamProvider(final InputStream stream, final java.nio.charset.Charset charset) { - this (new BufferedReader(new InputStreamReader(stream, charset))); + public StreamProvider(final InputStream stream, final java.nio.charset.Charset charset) + { + this (new BufferedReader (new InputStreamReader (stream, charset))); } - public StreamProvider(final Reader reader) { + public StreamProvider (final Reader reader) + { m_aReader = reader; } - public int read(final char[] aDest, final int nOfs, final int nLen) throws IOException { + public int read (final char[] aDest, final int nOfs, final int nLen) throws IOException + { int result = m_aReader.read(aDest, nOfs, nLen); /* CBA -- Added 2014/03/29 -- @@ -42,7 +47,9 @@ public class StreamProvider implements Provider { return result; } - public void close() throws IOException { - m_aReader.close(); + public void close () throws IOException + { + if (m_aReader != null) + m_aReader.close(); } } diff --git a/src/main/resources/templates/stream/java/modern/StringProvider.template b/src/main/resources/templates/stream/java/modern/StringProvider.template index a5cd96448..5cad61e8f 100644 --- a/src/main/resources/templates/stream/java/modern/StringProvider.template +++ b/src/main/resources/templates/stream/java/modern/StringProvider.template @@ -1,17 +1,20 @@ import java.io.IOException; -public class StringProvider implements Provider { +public class StringProvider implements Provider +{ private String m_sStr; private int m_nPos = 0; private final int m_nLen; - public StringProvider(final String sStr) { + public StringProvider(final String sStr) + { m_sStr = sStr; m_nLen = sStr.length(); } - public int read(final char[] aDest, final int nOfs, final int nLen) throws IOException { + public int read (final char[] aDest, final int nOfs, final int nLen) throws IOException + { final int nLeft = m_nLen - m_nPos; if (nLeft <= 0) return -1; @@ -28,7 +31,8 @@ public class StringProvider implements Provider { return nCharsRead; } - public void close() { + public void close() + { m_sStr = null; } -} \ No newline at end of file +} diff --git a/src/test/java/com/helger/pgcc/output/java/JavaTemplateValidityFuncTest.java b/src/test/java/com/helger/pgcc/output/java/JavaTemplateValidityFuncTest.java index 25d0632cd..080482e81 100644 --- a/src/test/java/com/helger/pgcc/output/java/JavaTemplateValidityFuncTest.java +++ b/src/test/java/com/helger/pgcc/output/java/JavaTemplateValidityFuncTest.java @@ -53,7 +53,7 @@ import com.helger.pgcc.parser.Options; /** - * Test class that creates the Java template files to disk and tries to compile + * Test class that creates the Java template files to disk and tries to parse * them with the JavaParser * * @author Philip Helger diff --git a/src/test/java/com/helger/pgcc/utils/ConditionParserTest.java b/src/test/java/com/helger/pgcc/utils/ConditionParserTest.java index bb2ff358f..9a79b7f1e 100644 --- a/src/test/java/com/helger/pgcc/utils/ConditionParserTest.java +++ b/src/test/java/com/helger/pgcc/utils/ConditionParserTest.java @@ -86,4 +86,14 @@ public void testBufferExpansion () throws ParseException final String b = StringHelper.getRepeated ('b', 4096 - 4 + 1); _test ("/*" + a + "*/\n/*" + b + "*/\nT || F", true); } + + @Test + public void testBufferExpansion2 () throws ParseException + { + // open + close of the comment + final String a = StringHelper.getRepeated ('a', 20480); + // force the buffer to expand and wrap around + final String b = StringHelper.getRepeated ('b', 1024 * 1024); + _test ("/*" + a + "*/\n/*" + b + "*/\nT || F", true); + } }