Escape character support for string literals

common/src/main/java/org/opensearch/sql/common/utils/StringUtils.java

@@ -12,52 +12,33 @@
 
 public class StringUtils {
   /**
-   * Unquote any string with mark specified.
-   * @param text string
-   * @param mark quotation mark
-   * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been
-   *     removed
-   */
-  public static String unquote(String text, String mark) {
-    if (isQuoted(text, mark)) {
-      return text.substring(mark.length(), text.length() - mark.length());
-    }
-    return text;
-  }
-
-  /**
-   * Unquote Identifier which has " or ' or ` as mark.
+   * Unquote Identifier which has " or ' as mark.
    * Strings quoted by ' or " with two of these quotes appearing next to each other in the quote
    * acts as an escape
    * Example: 'Test''s' will result in 'Test's', similar with those single quotes being replaced
-   * with double.
+   * with double quote.
+   * Supports escaping quotes (single/double) and escape characters using the `\` characters.
    * @param text string
-   * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been
+   * @return An unquoted string whose outer pair of (single/double) quotes have been
    *     removed
    */
   public static String unquoteText(String text) {
-
     if (text.length() < 2) {
       return text;
     }
 
-    char enclosingQuote;
+    char enclosingQuote = 0;
     char firstChar = text.charAt(0);
     char lastChar = text.charAt(text.length() - 1);
 
     if (firstChar == lastChar
         && (firstChar == '\''
-        || firstChar == '"'
-        || firstChar == '`')) {
+        || firstChar == '"')) {
       enclosingQuote = firstChar;
     } else {
       return text;
     }
 
-    if (enclosingQuote == '`') {
-      return text.substring(1, text.length() - 1);
-    }
-
     char currentChar;
     char nextChar;
 
@@ -67,13 +48,18 @@ public static String unquoteText(String text) {
     for (int chIndex = 1; chIndex < text.length() - 1; chIndex++) {
       currentChar = text.charAt(chIndex);
       nextChar = text.charAt(chIndex + 1);
-      if (currentChar == enclosingQuote
-          && nextChar == currentChar) {
+
+      if ((currentChar == '\\'
+          && (nextChar == '"'
+          || nextChar == '\\'
+          || nextChar == '\''))
+          || currentChar == nextChar
+          && currentChar == enclosingQuote) {
         chIndex++;
+        currentChar = nextChar;
       }
       textSB.append(currentChar);
     }
-
     return textSB.toString();
   }
 

core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java

@@ -10,39 +10,35 @@ class StringUtilsTest {
   void unquoteTest() {
     assertEquals("test", unquoteText("test"));
     assertEquals("test", unquoteText("'test'"));
-    assertEquals("test", unquoteText("`test`"));
 
     assertEquals("test'", unquoteText("'test'''"));
     assertEquals("test\"", unquoteText("\"test\"\"\""));
 
     assertEquals("te``st", unquoteText("'te``st'"));
     assertEquals("te``st", unquoteText("\"te``st\""));
-    assertEquals("te``st", unquoteText("`te``st`"));
 
     assertEquals("te'st", unquoteText("'te''st'"));
     assertEquals("te''st", unquoteText("\"te''st\""));
-    assertEquals("te''st", unquoteText("`te''st`"));
 
     assertEquals("te\"\"st", unquoteText("'te\"\"st'"));
     assertEquals("te\"st", unquoteText("\"te\"\"st\""));
-    assertEquals("te\"\"st", unquoteText("`te\"\"st`"));
 
     assertEquals("''", unquoteText("''''''"));
     assertEquals("\"\"", unquoteText("\"\"\"\"\"\""));
-    assertEquals("````", unquoteText("``````"));
 
     assertEquals("test'", unquoteText("'test''"));
 
     assertEquals("", unquoteText(""));
     assertEquals("'", unquoteText("'"));
-    assertEquals("`", unquoteText("`"));
     assertEquals("\"", unquoteText("\""));
 
     assertEquals("hello'", unquoteText("'hello''"));
     assertEquals("don't", unquoteText("'don't'"));
-    assertEquals("hello`", unquoteText("`hello``"));
     assertEquals("don\"t", unquoteText("\"don\"t\""));
 
+    assertEquals("hel\\lo'", unquoteText("'hel\\lo''"));
+    assertEquals("hel'lo", unquoteText("'hel'lo'"));
+    assertEquals("hel\"lo", unquoteText("\"hel\"lo\""));
+    assertEquals("hel\\'\\lo", unquoteText("'hel\\\\''\\\\lo'"));
   }
-
 }

docs/user/dql/expressions.rst

@@ -40,6 +40,15 @@ Here is an example for different type of literals::
     | 123   | hello     | False   | -4.567   | 2020-07-07          | 01:01:01          | 2020-07-07 01:01:01               |
     +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+
 
+
+    os> SELECT "Hello", 'Hello', "It""s", 'It''s', "It's", '"Its"', 'It\'s', 'It\\\'s', "\I\t\s"
+    fetched rows / total rows = 1/1
+    +-----------+-----------+-----------+-----------+----------+-----------+-----------+-------------+------------+
+    | "Hello"   | 'Hello'   | "It""s"   | 'It''s'   | "It's"   | '"Its"'   | 'It\'s'   | 'It\\\'s'   | "\I\t\s"   |
+    |-----------+-----------+-----------+-----------+----------+-----------+-----------+-------------+------------|
+    | Hello     | Hello     | It"s      | It's      | It's     | "Its"     | It's      | It\'s       | \I\t\s     |
+    +-----------+-----------+-----------+-----------+----------+-----------+-----------+-------------+------------+
+
 Limitations
 -----------
 

integ-test/src/test/java/org/opensearch/sql/ppl/MultiMatchIT.java

@@ -23,7 +23,7 @@ public void init() throws IOException {
   @Test
   public void test_multi_match() throws IOException {
     String query = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE multi_match([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste') | fields Id";
+        + " | WHERE multi_match([\\\"Tags\\\" ^ 1.5, Title, 'Body' 4.2], 'taste') | fields Id";
     var result = executeQuery(query);
     assertEquals(16, result.getInt("total"));
   }

integ-test/src/test/java/org/opensearch/sql/ppl/QueryStringIT.java

@@ -20,14 +20,14 @@ public void init() throws IOException {
 
   @Test
   public void all_fields_test() throws IOException {
-    String query = "source=" + TEST_INDEX_BEER + " | where query_string([`*`], 'taste')";
+    String query = "source=" + TEST_INDEX_BEER + " | where query_string(['*'], 'taste')";
     JSONObject result = executeQuery(query);
     assertEquals(16, result.getInt("total"));
   }
 
   @Test
   public void mandatory_params_test() throws IOException {
-    String query = "source=" + TEST_INDEX_BEER + " | where query_string([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste')";
+    String query = "source=" + TEST_INDEX_BEER + " | where query_string([\\\"Tags\\\" ^ 1.5, Title, 'Body' 4.2], 'taste')";
     JSONObject result = executeQuery(query);
     assertEquals(16, result.getInt("total"));
   }

integ-test/src/test/java/org/opensearch/sql/ppl/SimpleQueryStringIT.java

@@ -26,7 +26,7 @@ public void init() throws IOException {
   @Test
   public void test_simple_query_string() throws IOException {
     String query = "SOURCE=" + TEST_INDEX_BEER
-        + " | WHERE simple_query_string([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste') | fields Id";
+        + " | WHERE simple_query_string([\\\"Tags\\\" ^ 1.5, Title, 'Body' 4.2], 'taste') | fields Id";
     var result = executeQuery(query);
     assertEquals(16, result.getInt("total"));
   }

integ-test/src/test/java/org/opensearch/sql/sql/MultiMatchIT.java

@@ -6,8 +6,6 @@
 package org.opensearch.sql.sql;
 
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BEER;
-import static org.opensearch.sql.util.MatcherUtils.rows;
-import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
 
 import java.io.IOException;
 import org.json.JSONObject;
@@ -29,99 +27,38 @@ public void init() throws IOException {
   */
 
   @Test
-  public void test_mandatory_params() {
+  public void test_mandatory_params() throws IOException {
     String query = "SELECT Id FROM " + TEST_INDEX_BEER
-        + " WHERE multi_match([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste')";
+        + " WHERE multi_match([\\\"Tags\\\" ^ 1.5, Title, 'Body' 4.2], 'taste')";
     JSONObject result = executeJdbcRequest(query);
     assertEquals(16, result.getInt("total"));
   }
 
   @Test
-  public void test_all_params() {
+  public void test_all_params() throws IOException {
     String query = "SELECT Id FROM " + TEST_INDEX_BEER
         + " WHERE multi_match(['Body', Tags], 'taste beer', operator='and', analyzer=english,"
         + "auto_generate_synonyms_phrase_query=true, boost = 0.77, cutoff_frequency=0.33,"
         + "fuzziness = 'AUTO:1,5', fuzzy_transpositions = false, lenient = true, max_expansions = 25,"
         + "minimum_should_match = '2<-25% 9<-3', prefix_length = 7, tie_breaker = 0.3,"
         + "type = most_fields, slop = 2, zero_terms_query = 'ALL');";
-    JSONObject result = executeJdbcRequest(query);
+    var result = new JSONObject(executeQuery(query, "jdbc"));
     assertEquals(10, result.getInt("total"));
   }
 
   @Test
-  public void verify_wildcard_test() {
+  public void verify_wildcard_test() throws IOException {
     String query1 = "SELECT Id FROM " + TEST_INDEX_BEER
         + " WHERE multi_match(['Tags'], 'taste')";
-    JSONObject result1 = executeJdbcRequest(query1);
+    var result1 = new JSONObject(executeQuery(query1, "jdbc"));
     String query2 = "SELECT Id FROM " + TEST_INDEX_BEER
         + " WHERE multi_match(['T*'], 'taste')";
-    JSONObject result2 = executeJdbcRequest(query2);
+    var result2 = new JSONObject(executeQuery(query2, "jdbc"));
     assertNotEquals(result2.getInt("total"), result1.getInt("total"));
 
     String query = "SELECT Id FROM " + TEST_INDEX_BEER
         + " WHERE multi_match(['*Date'], '2014-01-22');";
-    JSONObject result = executeJdbcRequest(query);
+    var result = new JSONObject(executeQuery(query, "jdbc"));
     assertEquals(10, result.getInt("total"));
   }
-
-  @Test
-  public void test_multimatch_alternate_parameter_syntax() {
-    String query = "SELECT Tags FROM " + TEST_INDEX_BEER
-        + " WHERE multimatch('query'='taste', 'fields'='Tags')";
-    JSONObject result = executeJdbcRequest(query);
-    assertEquals(8, result.getInt("total"));
-  }
-
-  @Test
-  public void test_multimatchquery_alternate_parameter_syntax() {
-    String query = "SELECT Tags FROM " + TEST_INDEX_BEER
-        + " WHERE multimatchquery(query='cicerone', fields='Tags')";
-    JSONObject result = executeJdbcRequest(query);
-    assertEquals(2, result.getInt("total"));
-    verifyDataRows(result, rows("serving cicerone restaurants"),
-        rows("taste cicerone"));
-  }
-
-  @Test
-  public void test_quoted_multi_match_alternate_parameter_syntax() {
-    String query = "SELECT Tags FROM " + TEST_INDEX_BEER
-        + " WHERE multi_match('query'='cicerone', 'fields'='Tags')";
-    JSONObject result = executeJdbcRequest(query);
-    assertEquals(2, result.getInt("total"));
-    verifyDataRows(result, rows("serving cicerone restaurants"),
-        rows("taste cicerone"));
-  }
-
-  @Test
-  public void test_multi_match_alternate_parameter_syntax() {
-    String query = "SELECT Tags FROM " + TEST_INDEX_BEER
-        + " WHERE multi_match(query='cicerone', fields='Tags')";
-    JSONObject result = executeJdbcRequest(query);
-    assertEquals(2, result.getInt("total"));
-    verifyDataRows(result, rows("serving cicerone restaurants"),
-        rows("taste cicerone"));
-  }
-
-  @Test
-  public void test_wildcard_multi_match_alternate_parameter_syntax() {
-    String query = "SELECT Body FROM " + TEST_INDEX_BEER
-        + " WHERE multi_match(query='IPA', fields='B*') LIMIT 1";
-    JSONObject result = executeJdbcRequest(query);
-    verifyDataRows(result, rows("<p>I know what makes an IPA an IPA, but what are the unique" +
-        " characteristics of it's common variants? To be specific, the ones I'm interested in are Double IPA" +
-        " and Black IPA, but general differences between any other styles would be welcome too. </p>\n"));
-  }
-
-  @Test
-  public void test_all_params_multimatchquery_alternate_parameter_syntax() {
-    String query = "SELECT Id FROM " + TEST_INDEX_BEER
-        + " WHERE multimatchquery(query='cicerone', fields='Tags', 'operator'='or', analyzer=english,"
-        + "auto_generate_synonyms_phrase_query=true, boost = 0.77, cutoff_frequency=0.33,"
-        + "fuzziness = 'AUTO:1,5', fuzzy_transpositions = false, lenient = true, max_expansions = 25,"
-        + "minimum_should_match = '2<-25% 9<-3', prefix_length = 7, tie_breaker = 0.3,"
-        + "type = most_fields, slop = 2, zero_terms_query = 'ALL');";
-
-    JSONObject result = executeJdbcRequest(query);
-    assertEquals(2, result.getInt("total"));
-  }
 }

integ-test/src/test/java/org/opensearch/sql/sql/QueryStringIT.java

@@ -21,15 +21,15 @@ public void init() throws IOException {
   @Test
   public void all_fields_test() throws IOException {
     String query = "SELECT * FROM "
-        + TEST_INDEX_BEER + " WHERE query_string([`*`], 'taste')";
+        + TEST_INDEX_BEER + " WHERE query_string(['*'], 'taste')";
     JSONObject result = executeJdbcRequest(query);
     assertEquals(16, result.getInt("total"));
   }
 
   @Test
   public void mandatory_params_test() throws IOException {
     String query = "SELECT Id FROM "
-        + TEST_INDEX_BEER + " WHERE query_string([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste')";
+        + TEST_INDEX_BEER + " WHERE query_string([\\\"Tags\\\" ^ 1.5, Title, 'Body' 4.2], 'taste')";
     JSONObject result = executeJdbcRequest(query);
     assertEquals(16, result.getInt("total"));
   }

integ-test/src/test/java/org/opensearch/sql/sql/SimpleQueryStringIT.java

@@ -22,14 +22,14 @@ public void init() throws IOException {
   The 'beer.stackexchange' index is a dump of beer.stackexchange.com converted to the format which might be ingested by OpenSearch.
   This is a forum like StackOverflow with questions about beer brewing. The dump contains both questions, answers and comments.
   The reference query is:
-    select count(Id) from beer.stackexchange where simple_query_string(["Tags" ^ 1.5, Title, `Body` 4.2], 'taste') and Tags like '% % %' and Title like '%';
-  It filters out empty `Tags` and `Title`.
+    select count(Id) from beer.stackexchange where simple_query_string(["Tags" ^ 1.5, Title, 'Body' 4.2], 'taste') and Tags like '% % %' and Title like '%';
+  It filters out empty 'Tags' and 'Title'.
   */
 
   @Test
   public void test_mandatory_params() throws IOException {
     String query = "SELECT Id FROM "
-        + TEST_INDEX_BEER + " WHERE simple_query_string([\\\"Tags\\\" ^ 1.5, Title, `Body` 4.2], 'taste')";
+        + TEST_INDEX_BEER + " WHERE simple_query_string([\\\"Tags\\\" ^ 1.5, Title, 'Body' 4.2], 'taste')";
     var result = new JSONObject(executeQuery(query, "jdbc"));
     assertEquals(16, result.getInt("total"));
   }

integ-test/src/test/java/org/opensearch/sql/sql/StringLiteralIT.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+
+package org.opensearch.sql.sql;
+
+import org.json.JSONObject;
+import org.junit.Test;
+import org.opensearch.sql.legacy.SQLIntegTestCase;
+
+import java.io.IOException;
+
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.schema;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+import static org.opensearch.sql.util.MatcherUtils.verifySchema;
+
+
+
+public class StringLiteralIT extends SQLIntegTestCase {
+  @Test
+  public void testStringHelloSingleQuote() throws IOException {
+    JSONObject result =
+        executeJdbcRequest("select 'Hello'");
+    verifySchema(result,
+        schema("'Hello'", null, "keyword"));
+    verifyDataRows(result, rows("Hello"));
+  }
+
+  @Test
+  public void testStringHelloDoubleQuote() throws IOException {
+    JSONObject result =
+        executeJdbcRequest("select \\\"Hello\\\"");
+    verifySchema(result,
+        schema("\"Hello\"", null, "keyword"));
+    verifyDataRows(result, rows("Hello"));
+  }
+
+  @Test
+  public void testImStringDoubleDoubleQuoteEscape() throws IOException {
+    JSONObject result =
+        executeJdbcRequest("select \\\"I\\\"\\\"m\\\"");
+    verifySchema(result,
+        schema("\"I\"\"m\"", null, "keyword"));
+    verifyDataRows(result, rows("I\"m"));
+  }
+
+  @Test
+  public void testImStringDoubleSingleQuoteEscape() throws IOException {
+    JSONObject result =
+        executeJdbcRequest("select 'I''m'");
+    verifySchema(result,
+        schema("'I''m'", null, "keyword"));
+    verifyDataRows(result, rows("I'm"));
+  }
+
+  @Test
+  public void testImStringEscapedSingleQuote() throws IOException {
+    JSONObject result =
+        executeJdbcRequest("select 'I\\\\'m'");
+    verifySchema(result,
+        schema("'I\\'m'", null, "keyword"));
+    verifyDataRows(result, rows("I'm"));
+  }
+}