From d39cfe6aafd2ec8f1250f53a18a0c0ebe8cd6da9 Mon Sep 17 00:00:00 2001 From: uriel Date: Thu, 20 Aug 2015 21:09:15 +0200 Subject: [PATCH] change InputStream input reading to use UTF8. --- .../net/minidev/json/parser/JSONParser.java | 7 ++- .../json/parser/JSONParserInputStream.java | 46 +++------------ .../java/net/minidev/json/test/TestUtf8.java | 58 +++++++++++++++++++ 3 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 json-smart/src/test/java/net/minidev/json/test/TestUtf8.java diff --git a/json-smart/src/main/java/net/minidev/json/parser/JSONParser.java b/json-smart/src/main/java/net/minidev/json/parser/JSONParser.java index 721d6c2..81afb05 100644 --- a/json-smart/src/main/java/net/minidev/json/parser/JSONParser.java +++ b/json-smart/src/main/java/net/minidev/json/parser/JSONParser.java @@ -17,6 +17,7 @@ */ import java.io.InputStream; import java.io.Reader; +import java.io.UnsupportedEncodingException; public class JSONParser { /** @@ -235,7 +236,7 @@ public Object parse(Reader in, ContainerFactory containerFactory, ContentHandler * use to return Primitive Type, or String, Or JsonObject or JsonArray * generated by a ContainerFactory */ - public Object parse(InputStream in) throws ParseException { + public Object parse(InputStream in) throws ParseException, UnsupportedEncodingException { if (pSBintream == null) pSBintream = new JSONParserInputStream(mode); return pSBintream.parse(in); @@ -245,7 +246,7 @@ public Object parse(InputStream in) throws ParseException { * use to return Primitive Type, or String, Or JsonObject or JsonArray * generated by a ContainerFactory */ - public Object parse(InputStream in, ContainerFactory containerFactory) throws ParseException { + public Object parse(InputStream in, ContainerFactory containerFactory) throws ParseException, UnsupportedEncodingException { if (pSBintream == null) pSBintream = new JSONParserInputStream(mode); return pSBintream.parse(in, containerFactory); @@ -256,7 +257,7 @@ public Object parse(InputStream in, ContainerFactory containerFactory) throws Pa * generated by a ContainerFactory */ public Object parse(InputStream in, ContainerFactory containerFactory, ContentHandler handler) - throws ParseException { + throws ParseException, UnsupportedEncodingException { if (pSBintream == null) pSBintream = new JSONParserInputStream(mode); return pSBintream.parse(in, containerFactory, handler); diff --git a/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java b/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java index 0432655..f9f3c03 100644 --- a/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java +++ b/json-smart/src/main/java/net/minidev/json/parser/JSONParserInputStream.java @@ -15,19 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import static net.minidev.json.parser.ParseException.ERROR_UNEXPECTED_EOF; - -import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; /** * Parser for JSON text. Please note that JSONParser is NOT thread-safe. * * @author Uriel Chemouni */ -class JSONParserInputStream extends JSONParserStream { - private InputStream in; - +class JSONParserInputStream extends JSONParserReader { // len public JSONParserInputStream(int permissiveMode) { super(permissiveMode); @@ -37,7 +34,7 @@ public JSONParserInputStream(int permissiveMode) { * use to return Primitive Type, or String, Or JsonObject or JsonArray * generated by a ContainerFactory */ - public Object parse(InputStream in) throws ParseException { + public Object parse(InputStream in) throws ParseException, UnsupportedEncodingException { return parse(in, ContainerFactory.FACTORY_SIMPLE, ContentHandlerDumy.HANDLER); } @@ -45,45 +42,20 @@ public Object parse(InputStream in) throws ParseException { * use to return Primitive Type, or String, Or JsonObject or JsonArray * generated by a ContainerFactory */ - public Object parse(InputStream in, ContainerFactory containerFactory) throws ParseException { + public Object parse(InputStream in, ContainerFactory containerFactory) throws ParseException, UnsupportedEncodingException { return parse(in, containerFactory, ContentHandlerDumy.HANDLER); } /** * use to return Primitive Type, or String, Or JsonObject or JsonArray * generated by a ContainerFactory + * @throws UnsupportedEncodingException */ public Object parse(InputStream in, ContainerFactory containerFactory, ContentHandler handler) - throws ParseException { - // - this.in = in; + throws ParseException, UnsupportedEncodingException { + InputStreamReader i2 = new InputStreamReader(in, "utf8"); this.pos = -1; - return super.parse(containerFactory, handler); - } - - protected void read() throws IOException { - int i = in.read(); - c = (i == -1) ? (char) EOI : (char) i; - pos++; - // + return super.parse(i2, containerFactory, handler); } - protected void readS() throws IOException { - sb.append(c); - int i = in.read(); - if (i == -1) { - c = EOI; - } else { - c = (char) i; - pos++; - } - } - - protected void readNoEnd() throws ParseException, IOException { - int i = in.read(); - if (i == -1) - throw new ParseException(pos - 1, ERROR_UNEXPECTED_EOF, "EOF"); - c = (char) i; - // - } } diff --git a/json-smart/src/test/java/net/minidev/json/test/TestUtf8.java b/json-smart/src/test/java/net/minidev/json/test/TestUtf8.java new file mode 100644 index 0000000..f802958 --- /dev/null +++ b/json-smart/src/test/java/net/minidev/json/test/TestUtf8.java @@ -0,0 +1,58 @@ +package net.minidev.json.test; + +import java.io.ByteArrayInputStream; +import java.io.StringReader; + +import junit.framework.TestCase; +import net.minidev.json.JSONObject; +import net.minidev.json.JSONValue; + +public class TestUtf8 extends TestCase { + // Sinhalese language + static String[] nonLatinTexts = new String[] { "සිංහල ජාතිය", "日本語", "Русский", "فارسی", "한국어", "Հայերեն", "हिन्दी", "עברית", "中文", "አማርኛ", "മലയാളം", + "ܐܬܘܪܝܐ", "მარგალური" }; + + public void testString() throws Exception { + for (String nonLatinText : nonLatinTexts) { + String s = "{\"key\":\"" + nonLatinText + "\"}"; + JSONObject obj = (JSONObject) JSONValue.parse(s); + String v = (String) obj.get("key"); // result is incorrect + System.out.println(v); + assertEquals(v, nonLatinText); + } + } + + public void testReader() throws Exception { + for (String nonLatinText : nonLatinTexts) { + String s = "{\"key\":\"" + nonLatinText + "\"}"; + StringReader reader = new StringReader(s); + JSONObject obj = (JSONObject) JSONValue.parse(reader); + + String v = (String) obj.get("key"); // result is incorrect + System.out.println(v); + assertEquals(v, nonLatinText); + } + } + + public void testInputStream() throws Exception { + for (String nonLatinText : nonLatinTexts) { + String s = "{\"key\":\"" + nonLatinText + "\"}"; + ByteArrayInputStream bis = new ByteArrayInputStream(s.getBytes("utf8")); + JSONObject obj = (JSONObject) JSONValue.parse(bis); + String v = (String) obj.get("key"); // result is incorrect + System.out.println(v); + assertEquals(v, nonLatinText); + } + } + + public void testBytes() throws Exception { + for (String nonLatinText : nonLatinTexts) { + String s = "{\"key\":\"" + nonLatinText + "\"}"; + byte[] bs = s.getBytes("utf8"); + JSONObject obj = (JSONObject) JSONValue.parse(bs); + String v = (String) obj.get("key"); // result is incorrect + System.out.println(v); + assertEquals(v, nonLatinText); + } + } +}