Add support to optionally allow surrogate pair entities (#165)

src/main/java/com/ctc/wstx/api/ReaderConfig.java

@@ -138,6 +138,8 @@ public final class ReaderConfig
     final static int PROP_MAX_ENTITY_DEPTH = 68;
 
     final static int PROP_MAX_DTD_DEPTH = 69;
+
+    final static int PROP_RESOLVE_ENTITY_SURROGATE_PAIRS = 70;
 
     /*
     ////////////////////////////////////////////////
@@ -361,6 +363,8 @@ public final class ReaderConfig
                         PROP_UNDECLARED_ENTITY_RESOLVER);
         sProperties.put(WstxInputProperties.P_BASE_URL,
                         PROP_BASE_URL);
+        sProperties.put(WstxInputProperties.P_RESOLVE_ENTITY_SURROGATE_PAIRS,
+                        PROP_RESOLVE_ENTITY_SURROGATE_PAIRS);
         sProperties.put(WstxInputProperties.P_INPUT_PARSING_MODE,
                         PROP_INPUT_PARSING_MODE);
     }
@@ -418,6 +422,11 @@ public final class ReaderConfig
      * references
      */
     protected URL mBaseURL;
+
+    /**
+     * Resolve surrogate pairs of entities (2 code-points as one target character)
+     */
+    protected boolean mResolveEntitySurrogatePairs = Boolean.FALSE;
 
     /**
      * Parsing mode can be changed from the default xml compliant
@@ -583,6 +592,7 @@ public ReaderConfig createNonShared(SymbolTable sym)
         rc.mMaxEntityDepth = mMaxEntityDepth;
         rc.mMaxEntityCount = mMaxEntityCount;
         rc.mMaxDtdDepth = mMaxDtdDepth;
+        rc.mResolveEntitySurrogatePairs = mResolveEntitySurrogatePairs;
         if (mSpecialProperties != null) {
             int len = mSpecialProperties.length;
             Object[] specProps = new Object[len];
@@ -791,6 +801,10 @@ public XMLResolver getUndeclaredEntityResolver() {
     }
 
     public URL getBaseURL() { return mBaseURL; }
+
+    public boolean isResolvingEntitySurrogatePairs() {
+        return mResolveEntitySurrogatePairs;
+    }
 
     public WstxInputProperties.ParsingMode getInputParsingMode() {
         return mParsingMode;
@@ -1074,6 +1088,10 @@ public void setUndeclaredEntityResolver(XMLResolver r) {
     }
 
     public void setBaseURL(URL baseURL) { mBaseURL = baseURL; }
+
+    public void setResolveEntitySurrogatePairs(boolean resolveEntitySurrogatePairs) {
+        mResolveEntitySurrogatePairs = resolveEntitySurrogatePairs;
+    }
 
     public void setInputParsingMode(WstxInputProperties.ParsingMode mode) {
         mParsingMode = mode;
@@ -1533,6 +1551,8 @@ public Object getProperty(int id)
             return getUndeclaredEntityResolver();
         case PROP_BASE_URL:
             return getBaseURL();
+        case PROP_RESOLVE_ENTITY_SURROGATE_PAIRS:
+            return isResolvingEntitySurrogatePairs();
         case PROP_INPUT_PARSING_MODE:
             return getInputParsingMode();
 
@@ -1757,6 +1777,10 @@ public boolean setProperty(String propName, int id, Object value)
                 setBaseURL(u);
             }
             break;
+
+        case PROP_RESOLVE_ENTITY_SURROGATE_PAIRS:
+            setResolveEntitySurrogatePairs(ArgUtil.convertToBoolean(propName, value));
+            break;
 
         case PROP_INPUT_PARSING_MODE:
             setInputParsingMode((WstxInputProperties.ParsingMode) value);

src/main/java/com/ctc/wstx/api/WstxInputProperties.java

@@ -300,6 +300,13 @@ public final class WstxInputProperties
      * DTD subset).
      */
     public final static String P_BASE_URL = "com.ctc.wstx.baseURL";
+
+    /**
+     * Property of type {@link java.lang.Boolean}, that will allow parsing
+     * high unicode characters written by surrogate pairs (2 code points)
+     * Default set as Boolean.FALSE, because it is not a standard behavior
+     */
+    public final static String P_RESOLVE_ENTITY_SURROGATE_PAIRS = "com.ctc.wstx.resolveEntitySurrogatePairs";
 
     // // // Alternate parsing modes
 

src/main/java/com/ctc/wstx/sr/StreamScanner.java

@@ -1188,55 +1188,121 @@ protected int resolveSimpleEntity(boolean checkStd)
         if (c == '#') {
             c = buf[ptr++];
             int value = 0;
+            // Turns to 0 when first entity is read (must be a high surrogate char)
+            int pairValue = -1;
             int inputLen = mInputEnd;
-            if (c == 'x') { // hex
-                while (ptr < inputLen) {
-                    c = buf[ptr++];
-                    if (c == ';') {
-                        break;
-                    }
-                    value = value << 4;
-                    if (c <= '9' && c >= '0') {
-                        value += (c - '0');
-                    } else if (c >= 'a' && c <= 'f') {
-                        value += (10 + (c - 'a'));
-                    } else if (c >= 'A' && c <= 'F') {
-                        value += (10 + (c - 'A'));
-                    } else {
-                        mInputPtr = ptr; // so error points to correct char
-                        throwUnexpectedChar(c, "; expected a hex digit (0-9a-fA-F).");
+            boolean isValueHighSurrogate = false;
+
+            /*
+             * Loop used only to continue iteration to search surrogate pair.
+             * If simple entity, it does only one iteration
+             */
+            while (value == 0 
+                    || (mConfig.isResolvingEntitySurrogatePairs() 
+                            && value >= 0xD800 && value <= 0xDBFF && pairValue <= 0)) {
+
+                int tmpValue = pairValue >= 0 ? pairValue : value;
+
+                if (c == 'x') { // hex
+                    while (ptr < inputLen) {
+                        c = buf[ptr++];
+                        if (c == ';') {
+                            break;
+                        }
+
+                        tmpValue = tmpValue << 4;
+                        if (c <= '9' && c >= '0') {
+                            tmpValue += (c - '0');
+                        } else if (c >= 'a' && c <= 'f') {
+                            tmpValue += (10 + (c - 'a'));
+                        } else if (c >= 'A' && c <= 'F') {
+                            tmpValue += (10 + (c - 'A'));
+                        } else {
+                            mInputPtr = ptr; // so error points to correct char
+                            throwUnexpectedChar(c, "; expected a hex digit (0-9a-fA-F).");
+                        }
+                        /* Need to check for overflow; easiest to do right as
+                         * it happens...
+                         */
+                        if (tmpValue > MAX_UNICODE_CHAR) {
+                            reportUnicodeOverflow();
+                        }
                     }
-                    /* Need to check for overflow; easiest to do right as
-                     * it happens...
-                     */
-                    if (value > MAX_UNICODE_CHAR) {
-                        reportUnicodeOverflow();
+                } else { // numeric (decimal)
+                    while (c != ';') {
+                        if (c <= '9' && c >= '0') {
+                            tmpValue = (tmpValue * 10) + (c - '0');
+                            // Overflow?
+                            if (tmpValue > MAX_UNICODE_CHAR) {
+                                reportUnicodeOverflow();
+                            }
+
+                        } else {
+                            mInputPtr = ptr; // so error points to correct char
+                            throwUnexpectedChar(c, "; expected a decimal number.");
+                        }
+                        if (ptr >= inputLen) {
+                            break;
+                        }
+                        c = buf[ptr++];
                     }
                 }
-            } else { // numeric (decimal)
-                while (c != ';') {
-                    if (c <= '9' && c >= '0') {
-                        value = (value * 10) + (c - '0');
-                        // Overflow?
-                        if (value > MAX_UNICODE_CHAR) {
-                            reportUnicodeOverflow();
+
+                if (pairValue >= 0) {
+                    pairValue = tmpValue;
+                } else {
+                    value = tmpValue;
+                }
+
+                isValueHighSurrogate = value >= 0xD800 && value <= 0xDBFF;
+
+                /*
+                 * If resolving entity surrogate pairs enabled and if current entity
+                 * is in range of high surrogate value, try to find surrogate pair 
+                 */
+                if (isValueHighSurrogate && mConfig.isResolvingEntitySurrogatePairs() 
+                        && c == ';' && pairValue < 0 && ptr + 1 < inputLen) {
+                    c = buf[ptr++];
+
+                    if (c == '&' && ptr + 1 < inputLen) {
+                        c = buf[ptr++];
+
+                        if (c == '#' && ptr + 1 < inputLen) {
+                            c = buf[ptr++];
+                            pairValue = 0;
+                            continue;
+                        } else {
+                            reportNoSurrogatePair(value);
                         }
                     } else {
-                        mInputPtr = ptr; // so error points to correct char
-                        throwUnexpectedChar(c, "; expected a decimal number.");
+                        reportNoSurrogatePair(value);
                     }
-                    if (ptr >= inputLen) {
-                        break;
-                    }
-                    c = buf[ptr++];
+                } else if (isValueHighSurrogate 
+                        && mConfig.isResolvingEntitySurrogatePairs() 
+                        && ptr + 1 >= inputLen) {
+                    reportNoSurrogatePair(value);
                 }
             }
+
             /* We get here either if we got it all, OR if we ran out of
              * input in current buffer.
              */
             if (c == ';') { // got the full thing
                 mInputPtr = ptr;
-                validateChar(value);
+
+                if (mConfig.isResolvingEntitySurrogatePairs() && pairValue > 0) {
+                    /*
+                     * If pair value is not in range of low surrogate values, then throw an error
+                     */
+                    if (pairValue < 0xDC00 || pairValue > 0xDFFF) {
+                        reportNoSurrogatePair(value);
+                    }
+
+                    value = 0x10000 + (value - 0xD800) * 0x400 + (pairValue - 0xDC00);
+                } else {
+                    validateChar(value);
+                }
+
                 return value;
             }
 
@@ -2464,6 +2530,12 @@ private void reportIllegalChar(int value)
     {
         throwParseError("Illegal character entity: expansion character (code 0x{0}", Integer.toHexString(value), null);
     }
+
+    private void reportNoSurrogatePair(int highSurrogate)
+        throws XMLStreamException
+    {
+        throwParseError("Cannot find surrogate pair: high surrogate character (code 0x{0})", Integer.toHexString(highSurrogate), null);
+    }
 
     protected void verifyLimit(String type, long maxValue, long currentValue)
         throws XMLStreamException

src/test/java/org/codehaus/stax/test/BaseStaxTest.java

@@ -8,6 +8,8 @@
 import javax.xml.stream.*;
 import javax.xml.stream.events.XMLEvent;
 
+import com.ctc.wstx.api.WstxInputProperties;
+
 /* Latest updates:
  *
  * - 07-Sep-2007, TSa: Updating based on latest understanding of
@@ -275,6 +277,14 @@ protected static boolean setSupportExternalEntities(XMLInputFactory f, boolean s
             return false;
         }
     }
+
+    protected static void setResolveEntitySurrogatePairs(XMLInputFactory f, boolean state)
+            throws XMLStreamException
+    {   
+        Boolean b = state ? Boolean.TRUE : Boolean.FALSE;
+        f.setProperty(WstxInputProperties.P_RESOLVE_ENTITY_SURROGATE_PAIRS, b);
+        assertEquals(b, f.getProperty(WstxInputProperties.P_RESOLVE_ENTITY_SURROGATE_PAIRS));
+    }
 
     protected static void setResolver(XMLInputFactory f, XMLResolver resolver)
         throws XMLStreamException