Better tokenized support and testing for Lucene

all/src/test/scala/spec/SimpleHaloAndLuceneSpec.scala

@@ -233,6 +233,14 @@ class SimpleHaloAndLuceneSpec extends AsyncWordSpec with AsyncIOSpec with Matche
           distances should be(List(28.555228128634383.miles, 1316.1223938032729.miles))
         }
     }
+    "search using tokenized data" in {
+      Person.query
+        .filter(Person.search === "john 21")
+        .toList
+        .map { results =>
+          results.map(_.name) should be(List("John Doe"))
+        }
+    }
     "delete John" in {
       Person.delete(id1).map { deleted =>
         deleted should be(id1)
@@ -318,6 +326,7 @@ class SimpleHaloAndLuceneSpec extends AsyncWordSpec with AsyncIOSpec with Matche
     val ageLinks: IndexedLinks[Int, Person] = indexedLinks[Int]("age", _.toString, _.age)
     val tag: LuceneIndex[String, Person] = index("tag", _.tags.toList)
     val point: LuceneIndex[GeoPoint, Person] = index.one("point", _.point, sorted = true)
+    val search: LuceneIndex[String, Person] = index("search", doc => List(doc.name, doc.age.toString) ::: doc.tags.toList, tokenized = true)
   }
 
   object InitialSetupUpgrade extends DatabaseUpgrade {

lucene/src/main/scala/lightdb/lucene/LuceneIndex.scala

@@ -26,6 +26,10 @@ case class LuceneIndex[F, D <: Document[D]](fieldName: String,
 
   def ===(value: F): LuceneFilter[D] = is(value)
   def is(value: F): LuceneFilter[D] = LuceneFilter(() => value.json match {
+    case Str(s, _) if tokenized =>
+      val b = new BooleanQuery.Builder
+      s.split("\\s+").foreach(s => b.add(new TermQuery(new Term(fieldName, s)), BooleanClause.Occur.MUST))
+      b.build()
     case Str(s, _) => new TermQuery(new Term(fieldName, s))
     case json => throw new RuntimeException(s"Unsupported equality check: $json (${rw.definition})")
   })