Trying to fix #609

quickwit-oss · Aug 6, 2019 · ae2e4e9 · ae2e4e9
1 parent efd1af1
commit ae2e4e9
Show file tree

Hide file tree

Showing 5 changed files with 64 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,7 @@ Tantivy 0.11.0
 =====================
 
 - Added f64 field. Internally reuse u64 code the same way i64 does (@fdb-hiroshima)
+- Closes #609. Better handling of hyphens in query parser.
 
 Tantivy 0.10.1
 =====================

diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.10.1"
+version = "0.11.0"
 authors = ["Paul Masurel <[email protected]>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -87,7 +87,6 @@ travis-ci = { repository = "tantivy-search/tantivy" }
 [dev-dependencies.fail]
 features = ["failpoints"]
 
-
 # Following the "fail" crate best practises, we isolate
 # tests that define specific behavior in fail check points
 # in a different binary.
@@ -98,4 +97,4 @@ features = ["failpoints"]
 [[test]]
 name = "failpoints"
 path = "tests/failpoints/mod.rs"
-required-features = ["fail/failpoints"]
+required-features = ["fail/failpoints"]
diff --git a/src/indexer/index_writer.rs b/src/indexer/index_writer.rs
@@ -761,7 +761,6 @@ mod tests {
     use crate::Index;
     use crate::ReloadPolicy;
     use crate::Term;
-    use fail;
 
     #[test]
     fn test_operations_group() {

diff --git a/src/query/query_parser/query_grammar.rs b/src/query/query_parser/query_grammar.rs
@@ -16,19 +16,37 @@ parser! {
         ).map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
     }
 }
-
+//
+//parser! {
+//    fn word[I]()(I) -> String
+//    where [I: Stream<Item = char>] {
+//        many1(satisfy(|c: char| c.is_alphanumeric() || c=='.' || c=='-'))
+//               .and_then(|s: String| {
+//                   match s.as_str() {
+//                     "OR" => Err(StreamErrorFor::<I>::unexpected_static_message("OR")),
+//                     "AND" => Err(StreamErrorFor::<I>::unexpected_static_message("AND")),
+//                     "NOT" => Err(StreamErrorFor::<I>::unexpected_static_message("NOT")),
+//                     _ => Ok(s)
+//                   }
+//               })
+//    }
+//}
 parser! {
     fn word[I]()(I) -> String
     where [I: Stream<Item = char>] {
-        many1(satisfy(|c: char| c.is_alphanumeric() || c=='.'))
-               .and_then(|s: String| {
-                   match s.as_str() {
-                     "OR" => Err(StreamErrorFor::<I>::unexpected_static_message("OR")),
-                     "AND" => Err(StreamErrorFor::<I>::unexpected_static_message("AND")),
-                     "NOT" => Err(StreamErrorFor::<I>::unexpected_static_message("NOT")),
-                     _ => Ok(s)
-                   }
-               })
+        (
+            satisfy(|c: char| c.is_alphanumeric()),
+            many(satisfy(|c: char| !c.is_whitespace() && c!=':' && c != '(' && c != ')'))
+        )
+        .map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
+        .and_then(|s: String| {
+           match s.as_str() {
+             "OR" => Err(StreamErrorFor::<I>::unexpected_static_message("OR")),
+             "AND" => Err(StreamErrorFor::<I>::unexpected_static_message("AND")),
+             "NOT" => Err(StreamErrorFor::<I>::unexpected_static_message("NOT")),
+             _ => Ok(s)
+           }
+        })
     }
 }
 
@@ -225,6 +243,13 @@ mod test {
         assert!(parse_to_ast().parse(query).is_err());
     }
 
+    #[test]
+    fn test_parse_query_to_ast_hyphen() {
+        test_parse_query_to_ast_helper("\"www-form-encoded\"", "\"www-form-encoded\"");
+        test_parse_query_to_ast_helper("www-form-encoded", "\"www-form-encoded\"");
+        test_parse_query_to_ast_helper("www-form-encoded", "\"www-form-encoded\"");
+    }
+
     #[test]
     fn test_parse_query_to_ast_not_op() {
         assert_eq!(
@@ -272,13 +297,17 @@ mod test {
         test_parse_query_to_ast_helper("-abc:toto", "-(abc:\"toto\")");
         test_parse_query_to_ast_helper("abc:a b", "(abc:\"a\" \"b\")");
         test_parse_query_to_ast_helper("abc:\"a b\"", "abc:\"a b\"");
+        test_is_parse_err("abc +    ");
+    }
+
+    #[test]
+    fn test_parse_query_to_ast_range() {
         test_parse_query_to_ast_helper("foo:[1 TO 5]", "foo:[\"1\" TO \"5\"]");
         test_parse_query_to_ast_helper("[1 TO 5]", "[\"1\" TO \"5\"]");
         test_parse_query_to_ast_helper("foo:{a TO z}", "foo:{\"a\" TO \"z\"}");
         test_parse_query_to_ast_helper("foo:[1 TO toto}", "foo:[\"1\" TO \"toto\"}");
         test_parse_query_to_ast_helper("foo:[* TO toto}", "foo:[\"*\" TO \"toto\"}");
         test_parse_query_to_ast_helper("foo:[1 TO *}", "foo:[\"1\" TO \"*\"}");
         test_parse_query_to_ast_helper("foo:[1.1 TO *}", "foo:[\"1.1\" TO \"*\"}");
-        test_is_parse_err("abc +    ");
     }
 }
diff --git a/src/query/query_parser/query_parser.rs b/src/query/query_parser/query_parser.rs
@@ -18,7 +18,7 @@ use crate::schema::{FieldType, Term};
 use crate::tokenizer::TokenizerManager;
 use combine::Parser;
 use std::borrow::Cow;
-use std::num::{ParseIntError, ParseFloatError};
+use std::num::{ParseFloatError, ParseIntError};
 use std::ops::Bound;
 use std::str::FromStr;
 
@@ -676,7 +676,7 @@ mod test {
     }
 
     #[test]
-    pub fn test_parse_query_to_ast_disjunction() {
+    pub fn test_parse_query_to_ast_single_term() {
         test_parse_query_to_logical_ast_helper(
             "title:toto",
             "Term([0, 0, 0, 0, 116, 111, 116, 111])",
@@ -700,6 +700,10 @@ mod test {
                 .unwrap(),
             QueryParserError::AllButQueryForbidden
         );
+    }
+
+    #[test]
+    pub fn test_parse_query_to_ast_two_terms() {
         test_parse_query_to_logical_ast_helper(
             "title:a b",
             "(Term([0, 0, 0, 0, 97]) (Term([0, 0, 0, 0, 98]) \
@@ -712,6 +716,10 @@ mod test {
              (1, Term([0, 0, 0, 0, 98]))]\"",
             false,
         );
+    }
+
+    #[test]
+    pub fn test_parse_query_to_ast_ranges() {
         test_parse_query_to_logical_ast_helper(
             "title:[a TO b]",
             "(Included(Term([0, 0, 0, 0, 97])) TO \
@@ -879,4 +887,15 @@ mod test {
             true,
         );
     }
+
+    #[test]
+    pub fn test_query_parser_hyphen() {
+        test_parse_query_to_logical_ast_helper(
+            "title:www-form-encoded",
+            "\"[(0, Term([0, 0, 0, 0, 119, 119, 119])), \
+             (1, Term([0, 0, 0, 0, 102, 111, 114, 109])), \
+             (2, Term([0, 0, 0, 0, 101, 110, 99, 111, 100, 101, 100]))]\"",
+            false,
+        );
+    }
 }