add support for phrase slop in query language

query-grammar/Cargo.toml

@@ -14,4 +14,4 @@ edition = "2018"
 [dependencies]
 combine = {version="4", default-features=false, features=[] }
 once_cell = "1.7.2"
-regex ={ version = "1.5.4", default-features = false, features = ["std"] }
+regex ={ version = "1.5.4", default-features = false, features = ["std", "unicode"] }

query-grammar/src/query_grammar.rs

@@ -49,10 +49,10 @@ fn word<'a>() -> impl Parser<&'a str, Output = String> {
     (
         satisfy(|c: char| {
             !c.is_whitespace()
-                && !['-', '^', '`', ':', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
+                && !['-', '^', '`', ':', '{', '}', '"', '[', ']', '(', ')', '~'].contains(&c)
         }),
         many(satisfy(|c: char| {
-            !c.is_whitespace() && ![':', '^', '{', '}', '"', '[', ']', '(', ')'].contains(&c)
+            !c.is_whitespace() && ![':', '^', '{', '}', '"', '[', ']', '(', ')', '~'].contains(&c)
         })),
     )
         .map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
@@ -120,22 +120,35 @@ fn date_time<'a>() -> impl Parser<&'a str, Output = String> {
 
 fn term_val<'a>() -> impl Parser<&'a str, Output = String> {
     let phrase = char('"').with(many1(satisfy(|c| c != '"'))).skip(char('"'));
-    phrase.or(word())
+    negative_number().or(phrase.or(word()))
 }
 
 fn term_query<'a>() -> impl Parser<&'a str, Output = UserInputLiteral> {
-    let term_val_with_field = negative_number().or(term_val());
-    (field_name(), term_val_with_field).map(|(field_name, phrase)| UserInputLiteral {
-        field_name: Some(field_name),
-        phrase,
+    (field_name(), term_val(), matching_distance_val()).map(
+        |(field_name, phrase, matching_distance)| UserInputLiteral {
+            field_name: Some(field_name),
+            phrase,
+            matching_distance,
+        },
+    )
+}
+
+fn matching_distance_val<'a>() -> impl Parser<&'a str, Output = u32> {
+    let matching_distance = (char('~'), many1(digit())).map(|(_, distance): (_, String)| distance);
+    optional(matching_distance).map(|distance| match distance {
+        Some(d) => d.parse::<u32>().unwrap(),
+        _ => 0,
     })
 }
 
 fn literal<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
-    let term_default_field = term_val().map(|phrase| UserInputLiteral {
-        field_name: None,
-        phrase,
-    });
+    let term_default_field =
+        (term_val(), matching_distance_val()).map(|(phrase, matching_distance)| UserInputLiteral {
+            field_name: None,
+            phrase,
+            matching_distance,
+        });
+
     attempt(term_query())
         .or(term_default_field)
         .map(UserInputLeaf::from)
@@ -714,4 +727,19 @@ mod test {
         );
         test_is_parse_err("abc +    ");
     }
+
+    #[test]
+    fn test_matching_distance() {
+        assert!(parse_to_ast().parse("\"a b\"~").is_err());
+        assert!(parse_to_ast().parse("foo:\"a b\"~").is_err());
+        assert!(parse_to_ast().parse("\"a b\"^2~4").is_err());
+        assert!(parse_to_ast().parse("~").is_err());
+
+        test_parse_query_to_ast_helper("a~2", "\"a\"~2");
+        test_parse_query_to_ast_helper("\"a b\"~0", "\"a b\"");
+        test_parse_query_to_ast_helper("\"a b\"~1", "\"a b\"~1");
+        test_parse_query_to_ast_helper("\"a b\"~3", "\"a b\"~3");
+        test_parse_query_to_ast_helper("foo:\"a b\"~300", "\"foo\":\"a b\"~300");
+        test_parse_query_to_ast_helper("\"a b\"~300^2", "(\"a b\"~300)^2");
+    }
 }

query-grammar/src/user_input_ast.rs

@@ -40,14 +40,19 @@ impl Debug for UserInputLeaf {
 pub struct UserInputLiteral {
     pub field_name: Option<String>,
     pub phrase: String,
+    pub matching_distance: u32,
 }
 
 impl fmt::Debug for UserInputLiteral {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
-        match self.field_name {
-            Some(ref field_name) => write!(formatter, "\"{}\":\"{}\"", field_name, self.phrase),
-            None => write!(formatter, "\"{}\"", self.phrase),
+        if let Some(ref field) = self.field_name {
+            write!(formatter, "\"{}\":", field)?;
         }
+        write!(formatter, "\"{}\"", self.phrase)?;
+        if self.matching_distance > 0 {
+            write!(formatter, "~{}", self.matching_distance)?;
+        }
+        Ok(())
     }
 }
 

src/query/phrase_query/phrase_query.rs

@@ -40,7 +40,12 @@ impl PhraseQuery {
     /// Creates a new `PhraseQuery` given a list of terms and their offsets.
     ///
     /// Can be used to provide custom offset for each term.
-    pub fn new_with_offset(mut terms: Vec<(usize, Term)>) -> PhraseQuery {
+    pub fn new_with_offset(terms: Vec<(usize, Term)>) -> PhraseQuery {
+        PhraseQuery::new_with_offset_and_slop(terms, 0)
+    }
+
+    /// Creates a new `PhraseQuery` given a list of terms, their offsets and a slop
+    pub fn new_with_offset_and_slop(mut terms: Vec<(usize, Term)>, slop: u32) -> PhraseQuery {
         assert!(
             terms.len() > 1,
             "A phrase query is required to have strictly more than one term."
@@ -54,7 +59,7 @@ impl PhraseQuery {
         PhraseQuery {
             field,
             phrase_terms: terms,
-            slop: 0,
+            slop,
         }
     }
 

src/query/query_parser/logical_ast.rs

@@ -8,7 +8,7 @@ use crate::Score;
 #[derive(Clone)]
 pub enum LogicalLiteral {
     Term(Term),
-    Phrase(Vec<(usize, Term)>),
+    Phrase(Vec<(usize, Term)>, u32),
     Range {
         field: Field,
         value_type: Type,
@@ -74,7 +74,14 @@ impl fmt::Debug for LogicalLiteral {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
         match *self {
             LogicalLiteral::Term(ref term) => write!(formatter, "{:?}", term),
-            LogicalLiteral::Phrase(ref terms) => write!(formatter, "\"{:?}\"", terms),
+            LogicalLiteral::Phrase(ref terms, distance) => {
+                write!(formatter, "\"{:?}\"", terms)?;
+                if distance > 0 {
+                    write!(formatter, "~{:?}", distance)
+                } else {
+                    Ok(())
+                }
+            }
             LogicalLiteral::Range {
                 ref lower,
                 ref upper,

src/query/query_parser/query_parser.rs

@@ -405,6 +405,7 @@ impl QueryParser {
         field: Field,
         json_path: &str,
         phrase: &str,
+        matching_distance: u32,
     ) -> Result<Vec<LogicalLiteral>, QueryParserError> {
         let field_entry = self.schema.get_field_entry(field);
         let field_type = field_entry.field_type();
@@ -461,6 +462,7 @@ impl QueryParser {
                     field_name,
                     field,
                     phrase,
+                    matching_distance,
                     &text_analyzer,
                     index_record_option,
                 )?
@@ -626,7 +628,12 @@ impl QueryParser {
                     self.compute_path_triplets_for_literal(&literal)?;
                 let mut asts: Vec<LogicalAst> = Vec::new();
                 for (field, json_path, phrase) in term_phrases {
-                    for ast in self.compute_logical_ast_for_leaf(field, json_path, phrase)? {
+                    for ast in self.compute_logical_ast_for_leaf(
+                        field,
+                        json_path,
+                        phrase,
+                        literal.matching_distance,
+                    )? {
                         // Apply some field specific boost defined at the query parser level.
                         let boost = self.field_boost(field);
                         asts.push(LogicalAst::Leaf(Box::new(ast)).boost(boost));
@@ -670,9 +677,9 @@ impl QueryParser {
 fn convert_literal_to_query(logical_literal: LogicalLiteral) -> Box<dyn Query> {
     match logical_literal {
         LogicalLiteral::Term(term) => Box::new(TermQuery::new(term, IndexRecordOption::WithFreqs)),
-        LogicalLiteral::Phrase(term_with_offsets) => {
-            Box::new(PhraseQuery::new_with_offset(term_with_offsets))
-        }
+        LogicalLiteral::Phrase(term_with_offsets, distance) => Box::new(
+            PhraseQuery::new_with_offset_and_slop(term_with_offsets, distance),
+        ),
         LogicalLiteral::Range {
             field,
             value_type,
@@ -689,6 +696,7 @@ fn generate_literals_for_str(
     field_name: &str,
     field: Field,
     phrase: &str,
+    matching_distance: u32,
     text_analyzer: &TextAnalyzer,
     index_record_option: IndexRecordOption,
 ) -> Result<Option<LogicalLiteral>, QueryParserError> {
@@ -710,7 +718,7 @@ fn generate_literals_for_str(
             field_name.to_string(),
         ));
     }
-    Ok(Some(LogicalLiteral::Phrase(terms)))
+    Ok(Some(LogicalLiteral::Phrase(terms, matching_distance)))
 }
 
 fn generate_literals_for_json_object(
@@ -741,7 +749,7 @@ fn generate_literals_for_json_object(
             field_name.to_string(),
         ));
     }
-    logical_literals.push(LogicalLiteral::Phrase(terms));
+    logical_literals.push(LogicalLiteral::Phrase(terms, 0));
     Ok(logical_literals)
 }
 
@@ -1493,4 +1501,13 @@ mod test {
         assert_eq!(&super::locate_splitting_dots(r#"a\.b.c"#), &[4]);
         assert_eq!(&super::locate_splitting_dots(r#"a\..b.c"#), &[3, 5]);
     }
+
+    #[test]
+    pub fn test_phrase_matching_distance() {
+        test_parse_query_to_logical_ast_helper(
+            "\"a b\"~2",
+            r#"("[(0, Term(type=Str, field=0, "a")), (1, Term(type=Str, field=0, "b"))]"~2 "[(0, Term(type=Str, field=1, "a")), (1, Term(type=Str, field=1, "b"))]"~2)"#,
+            false,
+        );
+    }
 }