fix issue: boolean "AND" error

fix issue #8 (comment)

CHANGELOG.mdown

@@ -1,10 +1,21 @@
 # 
 
+## 0.8.9
+
+* fix issue [#8](https://github.com/weixsong/elasticlunr.js/issues/8#issuecomment-193266856), boolean "AND" error.
+* need to add test cases for issue fix #8
+
+## 0.8.8
+
+* during to the ugly slow network of my family, re-publish this version.
+
 ## 0.8.7
 
 * remove make server
 * remove .DS_Store
 * remove folder perf
+* fix issue [#8](https://github.com/weixsong/elasticlunr.js/issues/8), do tokenizenation even if the parameter of tokenizer is array type, because if we do not do segmentation, the whole array item will be used to build index then partial query will not be searchable.
+* but currently exist cross query error, need to fix.
 
 ## 0.8.6
 

VERSION

@@ -1 +1 @@
-0.8.6
+0.8.8

bower.json

@@ -1,6 +1,6 @@
 {
   "name": "elasticlunr.js",
-  "version": "0.8.6",
+  "version": "0.8.8",
   "main": "elasticlunr.js",
   "ignore": [
     "tests/",

component.json

@@ -1,7 +1,7 @@
 {
   "name": "elasticlunr",
   "repo": "weixsong/elasticlunr.js",
-  "version": "0.8.6",
+  "version": "0.8.8",
   "description": "Lightweight full-text search engine in Javascript for browser search and offline search.",
   "license": "MIT",
   "main": "elasticlunr.js",

docs/index.html

@@ -92,7 +92,7 @@ <h1>elasticlunr</h1>
         </div>
         <div class="span9">
           <section id="Main"><h1>Elasticlunr.js</h1>
-<p><a href="https://travis-ci.org/weixsong/elasticlunr.js"><img src="https://travis-ci.org/weixsong/elasticlunr.js.svg?branch=master" alt="Build Status"></a></p><p>Elasticlunr.js is a lightweight full-text search engine developed in JavaScript for browser search and offline search.<br />Elasticlunr.js is developed based on Lunr.js, but more flexible than lunr.js. Elasticlunr.js provides Query-Time boosting, field search, more rational scoring/ranking methodology, fast computation speed and so on.<br />Elasticlunr.js is a bit like Solr, but much smaller and not as bright, but also provide flexible configuration, query-time boosting, field search and other features.</p><h1>Why You Need Lightweight Offline Search?</h1>
+<p><a href="https://travis-ci.org/weixsong/elasticlunr.js"><img src="https://travis-ci.org/weixsong/elasticlunr.js.svg?branch=master" alt="Build Status"></a><br /><a href="https://badge.fury.io/js/elasticlunr"><img src="https://badge.fury.io/js/elasticlunr.svg" alt="npm version"></a><br /><a href="https://raw.githubusercontent.com/weixsong/elasticlunr.js/master/LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="GitHub license"></a></p><p>Elasticlunr.js is a lightweight full-text search engine developed in JavaScript for browser search and offline search.<br />Elasticlunr.js is developed based on Lunr.js, but more flexible than lunr.js. Elasticlunr.js provides Query-Time boosting, field search, more rational scoring/ranking methodology, fast computation speed and so on.<br />Elasticlunr.js is a bit like Solr, but much smaller and not as bright, but also provide flexible configuration, query-time boosting, field search and other features.</p><h1>Why You Need Lightweight Offline Search?</h1>
 <ol>
 <li>In some system, you don&#39;t want to deploy any <strong>complex full-text search engine</strong>(such as Lucence, Elasticsearch, Sphinx, etc.), you only want to provide some static web pages and provide search functionality , then you could build index in previous and load index in client side(such as Browser).</li>
 <li>Provide offline search functionality. For some documents, user usually download these documents, you could build index and put index in the documents package, then provide offline search functionality.</li>

docs/tokenizer.js.html

@@ -143,9 +143,17 @@ <h5 class="subheader"></h5>
       return true;
     });
 
-    return arr.map(function (t) {
+    arr = arr.map(function (t) {
       return elasticlunr.utils.toString(t).toLowerCase();
     });
+
+    var out = [];
+    arr.forEach(function(item) {
+      var tokens = item.split(elasticlunr.tokenizer.seperator);
+      out = out.concat(tokens);
+    }, this);
+
+    return out;
   }
 
   return str.toString().trim().toLowerCase().split(elasticlunr.tokenizer.seperator);

elasticlunr.js

@@ -1,6 +1,6 @@
 /**
  * elasticlunr - http://weixsong.github.io
- * Lightweight full-text search engine in Javascript for browser search and offline search. - 0.8.6
+ * Lightweight full-text search engine in Javascript for browser search and offline search. - 0.8.8
  *
  * Copyright (C) 2016 Oliver Nightingale
  * Copyright (C) 2016 Wei Song
@@ -83,7 +83,7 @@ var elasticlunr = function (config) {
   return idx;
 };
 
-elasticlunr.version = "0.8.6";
+elasticlunr.version = "0.8.8";
 /*!
  * elasticlunr.utils
  * Copyright (C) 2016 Oliver Nightingale
@@ -243,9 +243,17 @@ elasticlunr.tokenizer = function (str) {
       return true;
     });
 
-    return arr.map(function (t) {
+    arr = arr.map(function (t) {
       return elasticlunr.utils.toString(t).toLowerCase();
     });
+
+    var out = [];
+    arr.forEach(function(item) {
+      var tokens = item.split(elasticlunr.tokenizer.seperator);
+      out = out.concat(tokens);
+    }, this);
+
+    return out;
   }
 
   return str.toString().trim().toLowerCase().split(elasticlunr.tokenizer.seperator);

example/elasticlunr.js

@@ -1,6 +1,6 @@
 /**
  * elasticlunr - http://weixsong.github.io
- * Lightweight full-text search engine in Javascript for browser search and offline search. - 0.8.6
+ * Lightweight full-text search engine in Javascript for browser search and offline search. - 0.8.8
  *
  * Copyright (C) 2016 Oliver Nightingale
  * Copyright (C) 2016 Wei Song
@@ -83,7 +83,7 @@ var elasticlunr = function (config) {
   return idx;
 };
 
-elasticlunr.version = "0.8.6";
+elasticlunr.version = "0.8.8";
 /*!
  * elasticlunr.utils
  * Copyright (C) 2016 Oliver Nightingale
@@ -243,9 +243,17 @@ elasticlunr.tokenizer = function (str) {
       return true;
     });
 
-    return arr.map(function (t) {
+    arr = arr.map(function (t) {
       return elasticlunr.utils.toString(t).toLowerCase();
     });
+
+    var out = [];
+    arr.forEach(function(item) {
+      var tokens = item.split(elasticlunr.tokenizer.seperator);
+      out = out.concat(tokens);
+    }, this);
+
+    return out;
   }
 
   return str.toString().trim().toLowerCase().split(elasticlunr.tokenizer.seperator);

lib/index.js

@@ -380,10 +380,21 @@ elasticlunr.Index.prototype.fieldSearch = function (queryTokens, fieldName, conf
     if (expand == true) {
       tokens = this.index[fieldName].expandToken(token);
     }
- 
+
     tokens.forEach(function (key) {
       var docs = this.index[fieldName].getDocs(key);
       var idf = this.idf(key, fieldName);
+
+      // only record appeared token for retrieved documents for the 
+      // original token, not for expaned token.
+      // beause for doing coordNorm for a retrieved document, coordNorm only care how many
+      // query token appear in that document.
+      // so expanded token should not be added into docTokens, if added, this will pollute the 
+      // coordNorm
+      if (key == token) {
+        this.fieldSearchStats(docTokens, key, docs);
+      }
+
       for (var docRef in docs) {
         var tf = this.index[fieldName].getTermFrequency(key, docRef);
         var fieldLength = this.documentStore.getFieldLength(docRef, fieldName);
@@ -397,14 +408,6 @@ elasticlunr.Index.prototype.fieldSearch = function (queryTokens, fieldName, conf
           // currently I'm not sure if this penality is enough,
           // need to do verification
           penality = (1 - (key.length - token.length) / key.length) * 0.15;
-        } else {
-          // only record appeared token for retrieved documents for the 
-          // original token, not for expaned token.
-          // beause for doing coordNorm for a retrieved document, coordNorm only care how many
-          // query token appear in that document.
-          // so expanded token should not be added into docTokens, if added, this will pollute the 
-          // coordNorm
-          this.fieldSearchStats(docTokens, key, docs);
         }
 
         var score = tf * idf * fieldLengthNorm * penality;

lib/tokenizer.js

@@ -28,9 +28,17 @@ elasticlunr.tokenizer = function (str) {
       return true;
     });
 
-    return arr.map(function (t) {
+    arr = arr.map(function (t) {
       return elasticlunr.utils.toString(t).toLowerCase();
     });
+
+    var out = [];
+    arr.forEach(function(item) {
+      var tokens = item.split(elasticlunr.tokenizer.seperator);
+      out = out.concat(tokens);
+    }, this);
+
+    return out;
   }
 
   return str.toString().trim().toLowerCase().split(elasticlunr.tokenizer.seperator);

package.json

@@ -1,7 +1,7 @@
 {
   "name": "elasticlunr",
   "description": "Lightweight full-text search engine in Javascript for browser search and offline search.",
-  "version": "0.8.6",
+  "version": "0.8.8",
   "author": "Wei Song",
   "keywords": ["search", "text retrieval", "offline search", "full text search"],
   "homepage": "http://weixsong.github.io",

release/elasticlunr.js

@@ -1,6 +1,6 @@
 /**
  * elasticlunr - http://weixsong.github.io
- * Lightweight full-text search engine in Javascript for browser search and offline search. - 0.8.6
+ * Lightweight full-text search engine in Javascript for browser search and offline search. - 0.8.8
  *
  * Copyright (C) 2016 Oliver Nightingale
  * Copyright (C) 2016 Wei Song
@@ -83,7 +83,7 @@ var elasticlunr = function (config) {
   return idx;
 };
 
-elasticlunr.version = "0.8.6";
+elasticlunr.version = "0.8.8";
 /*!
  * elasticlunr.utils
  * Copyright (C) 2016 Oliver Nightingale
@@ -243,9 +243,17 @@ elasticlunr.tokenizer = function (str) {
       return true;
     });
 
-    return arr.map(function (t) {
+    arr = arr.map(function (t) {
       return elasticlunr.utils.toString(t).toLowerCase();
     });
+
+    var out = [];
+    arr.forEach(function(item) {
+      var tokens = item.split(elasticlunr.tokenizer.seperator);
+      out = out.concat(tokens);
+    }, this);
+
+    return out;
   }
 
   return str.toString().trim().toLowerCase().split(elasticlunr.tokenizer.seperator);

test/tokenizer_test.js

@@ -117,3 +117,27 @@ test("test get seperator function", function () {
   elasticlunr.tokenizer.setSeperator(sep2);
   deepEqual(elasticlunr.tokenizer.getSeperator(), sep2);
 });
+
+test("tokenize array", function () {
+  var str = ['hello world', 'glad to see you'];
+  var tokens = elasticlunr.tokenizer(str);
+  deepEqual(tokens, ['hello', 'world', 'glad', 'to', 'see', 'you']);
+});
+
+test("tokenize array 2", function () {
+  var str = ['helloworld', 'glad to see you'];
+  var tokens = elasticlunr.tokenizer(str);
+  deepEqual(tokens, ['helloworld', 'glad', 'to', 'see', 'you']);
+});
+
+test("tokenize array", function () {
+  var str = ['helloworld', null, undefined, 'glad to see you'];
+  var tokens = elasticlunr.tokenizer(str);
+  deepEqual(tokens, ['helloworld', 'glad', 'to', 'see', 'you']);
+});
+
+test("tokenize array", function () {
+  var str = ['helloworld', 'glad to see you', 'hyper-parameters'];
+  var tokens = elasticlunr.tokenizer(str);
+  deepEqual(tokens, ['helloworld', 'glad', 'to', 'see', 'you', 'hyper', 'parameters']);
+});