From 17bfe91cd5a87f3c0f7299b86686bde0075b2fc4 Mon Sep 17 00:00:00 2001
From: Abraham Polk <abepolk@gmail.com>
Date: Fri, 5 Apr 2019 10:40:30 -0400
Subject: [PATCH] QuotedStringTokenizer now does not unquote (#4830)

* QuotedStringTokenizer now does not unquote

* Unquote to fix test failures

* Add javadoc to nextToken
---
 .../jabref/logic/importer/util/GroupsParser.java | 16 ++++++++--------
 .../util/strings/QuotedStringTokenizer.java      |  5 +++++
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/jabref/logic/importer/util/GroupsParser.java b/src/main/java/org/jabref/logic/importer/util/GroupsParser.java
index f7f8dc27cdb..dd25ce3dc7a 100644
--- a/src/main/java/org/jabref/logic/importer/util/GroupsParser.java
+++ b/src/main/java/org/jabref/logic/importer/util/GroupsParser.java
@@ -169,8 +169,8 @@ private static AbstractGroup automaticKeywordGroupFromString(String string) {
         String name = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
         GroupHierarchyType context = GroupHierarchyType.getByNumberOrDefault(Integer.parseInt(tok.nextToken()));
         String field = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
-        Character delimiter = tok.nextToken().charAt(0);
-        Character hierarchicalDelimiter = tok.nextToken().charAt(0);
+        Character delimiter = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR).charAt(0);
+        Character hierarchicalDelimiter = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR).charAt(0);
         AutomaticKeywordGroup newGroup = new AutomaticKeywordGroup(name, context, field, delimiter, hierarchicalDelimiter);
         addGroupDetails(tok, newGroup);
         return newGroup;
@@ -212,7 +212,7 @@ private static ExplicitGroup explicitGroupFromString(String input, Character key
         QuotedStringTokenizer tok = new QuotedStringTokenizer(input.substring(MetadataSerializationConfiguration.EXPLICIT_GROUP_ID.length()),
                 MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR, MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
 
-        String name = tok.nextToken();
+        String name = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
         try {
             int context = Integer.parseInt(tok.nextToken());
             ExplicitGroup newGroup = new ExplicitGroup(name, GroupHierarchyType.getByNumberOrDefault(context), keywordSeparator);
@@ -230,7 +230,7 @@ private static ExplicitGroup legacyExplicitGroupFromString(String input, Charact
         QuotedStringTokenizer tok = new QuotedStringTokenizer(input.substring(MetadataSerializationConfiguration.LEGACY_EXPLICIT_GROUP_ID.length()),
                 MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR, MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
 
-        String name = tok.nextToken();
+        String name = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
         try {
             int context = Integer.parseInt(tok.nextToken());
             ExplicitGroup newGroup = new ExplicitGroup(name, GroupHierarchyType.getByNumberOrDefault(context), keywordSeparator);
@@ -274,15 +274,15 @@ private static AbstractGroup searchGroupFromString(String s) {
         QuotedStringTokenizer tok = new QuotedStringTokenizer(s.substring(MetadataSerializationConfiguration.SEARCH_GROUP_ID.length()),
                 MetadataSerializationConfiguration.GROUP_UNIT_SEPARATOR, MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
 
-        String name = tok.nextToken();
+        String name = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
         int context = Integer.parseInt(tok.nextToken());
-        String expression = tok.nextToken();
+        String expression = StringUtil.unquote(tok.nextToken(), MetadataSerializationConfiguration.GROUP_QUOTE_CHAR);
         boolean caseSensitive = Integer.parseInt(tok.nextToken()) == 1;
         boolean regExp = Integer.parseInt(tok.nextToken()) == 1;
         // version 0 contained 4 additional booleans to specify search
         // fields; these are ignored now, all fields are always searched
-        return new SearchGroup(StringUtil.unquote(name, MetadataSerializationConfiguration.GROUP_QUOTE_CHAR),
-                GroupHierarchyType.getByNumberOrDefault(context), StringUtil.unquote(expression, MetadataSerializationConfiguration.GROUP_QUOTE_CHAR), caseSensitive, regExp
+        return new SearchGroup(name,
+                GroupHierarchyType.getByNumberOrDefault(context), expression, caseSensitive, regExp
         );
     }
 
diff --git a/src/main/java/org/jabref/logic/util/strings/QuotedStringTokenizer.java b/src/main/java/org/jabref/logic/util/strings/QuotedStringTokenizer.java
index 720d6fe12e2..8c27e745dd4 100644
--- a/src/main/java/org/jabref/logic/util/strings/QuotedStringTokenizer.java
+++ b/src/main/java/org/jabref/logic/util/strings/QuotedStringTokenizer.java
@@ -34,6 +34,10 @@ public QuotedStringTokenizer(String content, String delimiters, char quoteCharac
         }
     }
 
+    /**
+     * @return the next token from the content string, ending at the next
+     * unquoted delimiter. Does not unquote the string itself.
+     */
     public String nextToken() {
         char c;
         StringBuilder stringBuilder = new StringBuilder();
@@ -41,6 +45,7 @@ public String nextToken() {
             c = content.charAt(index);
             if (c == quoteChar) { // next is quoted
                 ++index;
+                stringBuilder.append(c);
                 if (index < contentLength) {
                     stringBuilder.append(content.charAt(index));
                     // ignore for delimiter search!