Trying to fix #609 (#616)

CHANGELOG.md

@@ -2,6 +2,7 @@ Tantivy 0.11.0
 =====================
 
 - Added f64 field. Internally reuse u64 code the same way i64 does (@fdb-hiroshima)
+- Closes #609. Better handling of hyphens in query parser.
 
 Tantivy 0.10.1
 =====================

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tantivy"
-version = "0.10.1"
+version = "0.11.0"
 authors = ["Paul Masurel <[email protected]>"]
 license = "MIT"
 categories = ["database-implementations", "data-structures"]
@@ -86,7 +86,6 @@ travis-ci = { repository = "tantivy-search/tantivy" }
 [dev-dependencies.fail]
 features = ["failpoints"]
 
-
 # Following the "fail" crate best practises, we isolate
 # tests that define specific behavior in fail check points
 # in a different binary.
@@ -97,4 +96,4 @@ features = ["failpoints"]
 [[test]]
 name = "failpoints"
 path = "tests/failpoints/mod.rs"
-required-features = ["fail/failpoints"]
+required-features = ["fail/failpoints"]

src/query/query_parser/query_grammar.rs

@@ -20,15 +20,19 @@ parser! {
 parser! {
     fn word[I]()(I) -> String
     where [I: Stream<Item = char>] {
-        many1(satisfy(|c: char| c.is_alphanumeric() || c=='.'))
-               .and_then(|s: String| {
-                   match s.as_str() {
-                     "OR" => Err(StreamErrorFor::<I>::unexpected_static_message("OR")),
-                     "AND" => Err(StreamErrorFor::<I>::unexpected_static_message("AND")),
-                     "NOT" => Err(StreamErrorFor::<I>::unexpected_static_message("NOT")),
-                     _ => Ok(s)
-                   }
-               })
+        (
+            satisfy(|c: char| c.is_alphanumeric()),
+            many(satisfy(|c: char| !c.is_whitespace() && ![':', '{', '}', '"', '[', ']', '(',')'].contains(&c)))
+        )
+        .map(|(s1, s2): (char, String)| format!("{}{}", s1, s2))
+        .and_then(|s: String| {
+           match s.as_str() {
+             "OR" => Err(StreamErrorFor::<I>::unexpected_static_message("OR")),
+             "AND" => Err(StreamErrorFor::<I>::unexpected_static_message("AND")),
+             "NOT" => Err(StreamErrorFor::<I>::unexpected_static_message("NOT")),
+             _ => Ok(s)
+           }
+        })
     }
 }
 
@@ -115,9 +119,7 @@ parser! {
             )
          )
         .or(attempt(
-            range().map(UserInputAST::from)
-            )
-        )
+            range().map(UserInputAST::from)))
         .or(literal().map(|leaf| UserInputAST::Leaf(Box::new(leaf))))
     }
 }
@@ -225,6 +227,13 @@ mod test {
         assert!(parse_to_ast().parse(query).is_err());
     }
 
+    #[test]
+    fn test_parse_query_to_ast_hyphen() {
+        test_parse_query_to_ast_helper("\"www-form-encoded\"", "\"www-form-encoded\"");
+        test_parse_query_to_ast_helper("www-form-encoded", "\"www-form-encoded\"");
+        test_parse_query_to_ast_helper("www-form-encoded", "\"www-form-encoded\"");
+    }
+
     #[test]
     fn test_parse_query_to_ast_not_op() {
         assert_eq!(
@@ -272,13 +281,17 @@ mod test {
         test_parse_query_to_ast_helper("-abc:toto", "-(abc:\"toto\")");
         test_parse_query_to_ast_helper("abc:a b", "(abc:\"a\" \"b\")");
         test_parse_query_to_ast_helper("abc:\"a b\"", "abc:\"a b\"");
+        test_is_parse_err("abc +    ");
+    }
+
+    #[test]
+    fn test_parse_query_to_ast_range() {
         test_parse_query_to_ast_helper("foo:[1 TO 5]", "foo:[\"1\" TO \"5\"]");
         test_parse_query_to_ast_helper("[1 TO 5]", "[\"1\" TO \"5\"]");
         test_parse_query_to_ast_helper("foo:{a TO z}", "foo:{\"a\" TO \"z\"}");
         test_parse_query_to_ast_helper("foo:[1 TO toto}", "foo:[\"1\" TO \"toto\"}");
         test_parse_query_to_ast_helper("foo:[* TO toto}", "foo:[\"*\" TO \"toto\"}");
         test_parse_query_to_ast_helper("foo:[1 TO *}", "foo:[\"1\" TO \"*\"}");
         test_parse_query_to_ast_helper("foo:[1.1 TO *}", "foo:[\"1.1\" TO \"*\"}");
-        test_is_parse_err("abc +    ");
     }
 }

src/query/query_parser/query_parser.rs

@@ -690,7 +690,7 @@ mod test {
     }
 
     #[test]
-    pub fn test_parse_query_to_ast_disjunction() {
+    pub fn test_parse_query_to_ast_single_term() {
         test_parse_query_to_logical_ast_helper(
             "title:toto",
             "Term([0, 0, 0, 0, 116, 111, 116, 111])",
@@ -714,6 +714,10 @@ mod test {
                 .unwrap(),
             QueryParserError::AllButQueryForbidden
         );
+    }
+
+    #[test]
+    pub fn test_parse_query_to_ast_two_terms() {
         test_parse_query_to_logical_ast_helper(
             "title:a b",
             "(Term([0, 0, 0, 0, 97]) (Term([0, 0, 0, 0, 98]) \
@@ -726,6 +730,10 @@ mod test {
              (1, Term([0, 0, 0, 0, 98]))]\"",
             false,
         );
+    }
+
+    #[test]
+    pub fn test_parse_query_to_ast_ranges() {
         test_parse_query_to_logical_ast_helper(
             "title:[a TO b]",
             "(Included(Term([0, 0, 0, 0, 97])) TO \
@@ -893,4 +901,15 @@ mod test {
             true,
         );
     }
+
+    #[test]
+    pub fn test_query_parser_hyphen() {
+        test_parse_query_to_logical_ast_helper(
+            "title:www-form-encoded",
+            "\"[(0, Term([0, 0, 0, 0, 119, 119, 119])), \
+             (1, Term([0, 0, 0, 0, 102, 111, 114, 109])), \
+             (2, Term([0, 0, 0, 0, 101, 110, 99, 111, 100, 101, 100]))]\"",
+            false,
+        );
+    }
 }