Preliminary implementation of Geo support

build.sbt

@@ -65,7 +65,7 @@ val mapdbVersion: String = "3.1.0"
 
 val jedisVersion: String = "5.1.5"
 
-val fabricVersion: String = "1.15.3"
+val fabricVersion: String = "1.15.4"
 
 val scribeVersion: String = "3.15.0"
 

core/src/main/scala/lightdb/Field.scala

@@ -123,7 +123,7 @@ object Field {
       case _ => false
     })
     case DefType.Opt(d) => string2Json(name, s, d)
-    case DefType.Enum(_) => str(s)
+    case DefType.Enum(_, _) => str(s)
     case DefType.Arr(d) => arr(s.split(";;").toList.map(string2Json(name, _, d)): _*)
     case _ => try {
       JsonParser(s)

core/src/main/scala/lightdb/Query.scala

@@ -93,7 +93,7 @@ case class Query[Doc <: Document[Doc], Model <: DocumentModel[Doc]](collection:
       apply(Conversion.Materialized(fields))
     }
 
-    def distance(f: Model => Field[Doc, Option[Geo.Point]],
+    def distance[G <: Geo](f: Model => Field[Doc, Option[G]],
                  from: Geo.Point,
                  sort: Boolean = true,
                  radius: Option[Distance] = None)
@@ -121,7 +121,7 @@ case class Query[Doc <: Document[Doc], Model <: DocumentModel[Doc]](collection:
   def count(implicit transaction: Transaction[Doc]): Int = copy(limit = Some(1), countTotal = true)
     .search.docs.total.get
 
-  protected def distanceSearch(field: Field[Doc, Option[Geo.Point]],
+  protected def distanceSearch[G <: Geo](field: Field[Doc, Option[G]],
                                from: Geo.Point,
                                sort: Boolean, radius: Option[Distance])
                               (implicit transaction: Transaction[Doc]): SearchResults[Doc, DistanceAndDoc[Doc]] = {

core/src/main/scala/lightdb/Sort.scala

@@ -22,17 +22,17 @@ object Sort {
     def desc: ByField[Doc, F] = direction(SortDirection.Descending)
   }
 
-  case class ByDistance[Doc <: Document[Doc]](field: Field[Doc, Option[Geo.Point]],
+  case class ByDistance[Doc <: Document[Doc], G <: Geo](field: Field[Doc, Option[G]],
                              from: Geo.Point,
                              direction: SortDirection = SortDirection.Ascending) extends Sort {
-    def direction(direction: SortDirection): ByDistance[Doc] = copy(direction = direction)
+    def direction(direction: SortDirection): ByDistance[Doc, G] = copy(direction = direction)
 
-    def ascending: ByDistance[Doc] = direction(SortDirection.Ascending)
+    def ascending: ByDistance[Doc, G] = direction(SortDirection.Ascending)
 
-    def asc: ByDistance[Doc] = direction(SortDirection.Ascending)
+    def asc: ByDistance[Doc, G] = direction(SortDirection.Ascending)
 
-    def descending: ByDistance[Doc] = direction(SortDirection.Descending)
+    def descending: ByDistance[Doc, G] = direction(SortDirection.Descending)
 
-    def desc: ByDistance[Doc] = direction(SortDirection.Descending)
+    def desc: ByDistance[Doc, G] = direction(SortDirection.Descending)
   }
 }

core/src/main/scala/lightdb/filter/Filter.scala

@@ -72,7 +72,7 @@ object Filter {
   }
 
   case class Distance[Doc <: Document[Doc]](fieldName: String, from: Geo.Point, radius: lightdb.distance.Distance) extends Filter[Doc] {
-    def field(model: DocumentModel[Doc]): Field[Doc, Geo.Point] = model.fieldByName(fieldName)
+    def field(model: DocumentModel[Doc]): Field[Doc, Geo] = model.fieldByName(fieldName)
     override lazy val fieldNames: List[String] = List(fieldName)
   }
 

core/src/main/scala/lightdb/spatial/Geo.scala

@@ -2,24 +2,63 @@ package lightdb.spatial
 
 import fabric.rw._
 
-sealed trait Geo
+sealed trait Geo {
+  def center: Geo.Point
+}
 
 object Geo {
   implicit val pRW: RW[Point] = RW.gen
-  implicit val mpRW: RW[MultiPoint] = RW.gen
-  implicit val lsRW: RW[LineString] = RW.gen
-  implicit val mlsRW: RW[MultiLineString] = RW.gen
-  implicit val plyRW: RW[Polygon] = RW.gen
-  implicit val mplyRW: RW[MultiPolygon] = RW.gen
+  private implicit val mpRW: RW[MultiPoint] = RW.gen
+  private implicit val lsRW: RW[Line] = RW.gen
+  private implicit val mlsRW: RW[MultiLine] = RW.gen
+  private implicit val plyRW: RW[Polygon] = RW.gen
+  private implicit val mplyRW: RW[MultiPolygon] = RW.gen
 
-  implicit val rw: RW[Geo] = RW.poly[Geo]()(
+  implicit val rw: RW[Geo] = RW.poly[Geo](className = Some("lightdb.spatial.Geo"))(
     pRW, mpRW, lsRW, mlsRW, plyRW, mplyRW
   )
 
-  case class Point(latitude: Double, longitude: Double) extends Geo
-  case class MultiPoint(points: List[Point]) extends Geo
-  case class LineString(points: List[Point]) extends Geo
-  case class MultiLineString(lines: List[LineString]) extends Geo
-  case class Polygon(points: List[Point]) extends Geo
-  case class MultiPolygon(polygons: List[Polygon]) extends Geo
+  def min(points: List[Point]): Point = {
+    val latitude = points.map(_.latitude).min
+    val longitude = points.map(_.longitude).min
+    Point(latitude, longitude)
+  }
+
+  def max(points: List[Point]): Point = {
+    val latitude = points.map(_.latitude).max
+    val longitude = points.map(_.longitude).max
+    Point(latitude, longitude)
+  }
+
+  def center(points: List[Point]): Point = {
+    val min = this.min(points)
+    val max = this.max(points)
+    val latitude = min.latitude + (max.latitude - min.latitude)
+    val longitude = min.longitude + (max.longitude - min.longitude)
+    Point(latitude, longitude)
+  }
+
+  case class Point(latitude: Double, longitude: Double) extends Geo {
+    override def center: Point = this
+  }
+  case class MultiPoint(points: List[Point]) extends Geo {
+    lazy val center: Point = Geo.center(points)
+  }
+  case class Line(points: List[Point]) extends Geo {
+    lazy val center: Point = Geo.center(points)
+  }
+  case class MultiLine(lines: List[Line]) extends Geo {
+    lazy val center: Point = Geo.center(lines.flatMap(_.points))
+  }
+  case class Polygon(points: List[Point]) extends Geo {
+    lazy val center: Point = Geo.center(points)
+  }
+  object Polygon {
+    def lonLat(points: Double*): Polygon = Polygon(points.grouped(2).map { p =>
+      Point(p.last, p.head)
+    }.toList)
+  }
+  case class MultiPolygon(polygons: List[Polygon]) extends Geo {
+    lazy val center: Point = Geo.center(polygons.flatMap(_.points))
+  }
 }

core/src/main/scala/lightdb/spatial/Spatial.scala

@@ -7,9 +7,9 @@ import org.locationtech.spatial4j.distance.DistanceUtils
 object Spatial {
   private lazy val context = SpatialContext.GEO
 
-  def distance(p1: Geo.Point, p2: Geo.Point): Distance = {
-    val point1 = context.getShapeFactory.pointLatLon(p1.latitude, p2.longitude)
-    val point2 = context.getShapeFactory.pointLatLon(p2.latitude, p2.longitude)
+  def distance(p1: Geo, p2: Geo): Distance = {
+    val point1 = context.getShapeFactory.pointLatLon(p1.center.latitude, p2.center.longitude)
+    val point2 = context.getShapeFactory.pointLatLon(p2.center.latitude, p2.center.longitude)
     val degrees = context.calcDistance(point1, point2)
     val distance = DistanceUtils.degrees2Dist(degrees, DistanceUtils.EARTH_MEAN_RADIUS_KM)
     distance.kilometers

core/src/main/scala/lightdb/store/Conversion.scala

@@ -18,7 +18,7 @@ object Conversion {
 
   case class Converted[Doc <: Document[Doc], T](f: Doc => T) extends Conversion[Doc, T]
 
-  case class Distance[Doc <: Document[Doc]](field: Field[Doc, Option[Geo.Point]],
+  case class Distance[Doc <: Document[Doc], G <: Geo](field: Field[Doc, Option[G]],
                            from: Geo.Point,
                            sort: Boolean,
                            radius: Option[lightdb.distance.Distance]) extends Conversion[Doc, DistanceAndDoc[Doc]]

core/src/test/scala/spec/AbstractSpatialSpec.scala

@@ -26,27 +26,39 @@ abstract class AbstractSpatialSpec extends AnyWordSpec with Matchers { spec =>
   private val oklahomaCity = Geo.Point(35.5514, -97.4075)
   private val yonkers = Geo.Point(40.9461, -73.8669)
 
+  private val moorePolygon = Geo.Polygon.lonLat(
+    -97.51995284659067, 35.31659661477283,
+    -97.50983688600051, 35.29708140953622,
+    -97.42966767585344, 35.29494585205129,
+    -97.41303352647198, 35.31020363480967,
+    -97.41331385837709, 35.34926895467585,
+    -97.42803670547956, 35.36508604748108,
+    -97.50690451974124, 35.36587866914906,
+    -97.51755160616675, 35.35131024794894,
+    -97.51995284659067, 35.31659661477283
+  )
+
   protected def supportsAggregateFunctions: Boolean = true
 
   private val p1 = Person(
     name = "John Doe",
     age = 21,
     point = newYorkCity,
-    altPoint = None,
+    geo = None,
     _id = id1
   )
   private val p2 = Person(
     name = "Jane Doe",
     age = 19,
     point = noble,
-    altPoint = Some(oklahomaCity),
+    geo = Some(moorePolygon),
     _id = id2
   )
   private val p3 = Person(
     name = "Bob Dole",
     age = 123,
     point = yonkers,
-    altPoint = Some(chicago),
+    geo = Some(chicago),
     _id = id3
   )
 
@@ -80,14 +92,14 @@ abstract class AbstractSpatialSpec extends AnyWordSpec with Matchers { spec =>
     "sort by distance from Noble using altPoint" in {
       DB.people.transaction { implicit transaction =>
         val list = DB.people.query.search.distance(
-          _.altPoint,
+          _.geo,
           from = noble,
           radius = Some(10_000.miles)
         ).iterator.toList
         val people = list.map(_.doc)
         val distances = list.map(_.distance.get.mi)
         people.map(_.name) should be(List("Jane Doe", "Bob Dole"))
-        distances should (be (List(28.307644231281916, 460.868070665109)) or be(List(28.307644231281916, 460.868070665109)))
+        distances should (be (List(15.489309276333513, 460.868070665109)) or be(List(28.307644231281916, 460.868070665109)))
       }
     }
     "truncate the database" in {
@@ -113,7 +125,7 @@ abstract class AbstractSpatialSpec extends AnyWordSpec with Matchers { spec =>
   case class Person(name: String,
                     age: Int,
                     point: Geo.Point,
-                    altPoint: Option[Geo.Point],
+                    geo: Option[Geo],
                     _id: Id[Person] = Person.id()) extends Document[Person]
 
   object Person extends DocumentModel[Person] with JsonConversion[Person] {
@@ -122,6 +134,6 @@ abstract class AbstractSpatialSpec extends AnyWordSpec with Matchers { spec =>
     val name: F[String] = field("name", _.name)
     val age: F[Int] = field("age", _.age)
     val point: I[Geo.Point] = field.index("point", _.point)
-    val altPoint: I[Option[Geo.Point]] = field.index("altPoint", _.altPoint)
+    val geo: I[Option[Geo]] = field.index("geo", _.geo)
   }
 }

lucene/src/main/scala/lightdb/lucene/LuceneStore.scala

@@ -16,7 +16,8 @@ import lightdb.spatial.{DistanceAndDoc, Geo, Spatial}
 import lightdb.store.{Conversion, Store, StoreManager, StoreMode}
 import lightdb.transaction.Transaction
 import lightdb.util.Aggregator
-import org.apache.lucene.document.{DoubleField, DoublePoint, IntField, IntPoint, LatLonDocValuesField, LatLonPoint, LongField, LongPoint, NumericDocValuesField, SortedDocValuesField, SortedNumericDocValuesField, StoredField, StringField, TextField, Document => LuceneDocument, Field => LuceneField}
+import org.apache.lucene.document.{DoubleField, DoublePoint, IntField, IntPoint, LatLonDocValuesField, LatLonPoint, LatLonShape, LongField, LongPoint, NumericDocValuesField, SortedDocValuesField, SortedNumericDocValuesField, StoredField, StringField, TextField, Document => LuceneDocument, Field => LuceneField}
+import org.apache.lucene.geo.{Line, Polygon}
 import org.apache.lucene.search.{BooleanClause, BooleanQuery, BoostQuery, FieldExistsQuery, IndexSearcher, MatchAllDocsQuery, RegexpQuery, ScoreDoc, SearcherFactory, SearcherManager, SortField, SortedNumericSortField, TermQuery, TopFieldCollector, TopFieldCollectorManager, TopFieldDocs, Query => LuceneQuery, Sort => LuceneSort}
 import org.apache.lucene.index.{StoredFields, Term}
 import org.apache.lucene.queryparser.classic.QueryParser
@@ -51,6 +52,40 @@ class LuceneStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory:
     addDoc(id(doc), luceneFields, upsert = true)
   }
 
+  private def createGeoFields(className: String,
+                              field: Field[Doc, _],
+                              json: Json,
+                              add: LuceneField => Unit): Unit = {
+    className match {
+      case "lightdb.spatial.Geo.Point" =>
+        val p = json.as[Geo.Point]
+        add(new LatLonPoint(field.name, p.latitude, p.longitude))
+      case _ =>
+        // Treat everything else like a LatLonShape (LatLonShape.createIndexableFields("", p.latitude, p.longitude))
+        def indexPoint(p: Geo.Point): Unit = LatLonShape.createIndexableFields(field.name, p.latitude, p.longitude)
+        def indexLine(l: Geo.Line): Unit = {
+          val line = new Line(l.points.map(_.latitude).toArray, l.points.map(_.longitude).toArray)
+          LatLonShape.createIndexableFields(field.name, line)
+        }
+        def indexPolygon(p: Geo.Polygon): Unit = {
+          def convert(p: Geo.Polygon): Polygon =
+            new Polygon(p.points.map(_.latitude).toArray, p.points.map(_.longitude).toArray)
+          convert(p)
+        }
+        val geo = json.as[Geo]
+        geo match {
+          case p: Geo.Point => indexPoint(p)
+          case Geo.MultiPoint(points) => points.foreach(indexPoint)
+          case l: Geo.Line => indexLine(l)
+          case Geo.MultiLine(lines) => lines.foreach(indexLine)
+          case p: Geo.Polygon => indexPolygon(p)
+          case Geo.MultiPolygon(polygons) => polygons.foreach(indexPolygon)
+        }
+        add(new LatLonPoint(field.name, geo.center.latitude, geo.center.longitude))
+    }
+    add(new StoredField(field.name, JsonFormatter.Compact(json)))
+  }
+
   private def createLuceneFields(field: Field[Doc, _], doc: Doc): List[LuceneField] = {
     def fs: LuceneField.Store = if (storeMode == StoreMode.All || field.indexed) LuceneField.Store.YES else LuceneField.Store.NO
     val json = field.getJson(doc)
@@ -60,29 +95,31 @@ class LuceneStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory:
       case t: Tokenized[Doc] =>
         List(new LuceneField(field.name, t.get(doc), if (fs == LuceneField.Store.YES) TextField.TYPE_STORED else TextField.TYPE_NOT_STORED))
       case _ =>
-        def addJson(json: Json, d: DefType): Unit = d match {
-          case DefType.Opt(DefType.Obj(_, Some("lightdb.spatial.Geo.Point"))) => json match {
-            case Null => // Don't set anything
-            case _ =>
-              val p = json.as[Geo.Point]
-              add(new LatLonPoint(field.name, p.latitude, p.longitude))
-              add(new StoredField(field.name, JsonFormatter.Compact(p.json)))
+        def addJson(json: Json, d: DefType): Unit = {
+          val className = d match {
+            case DefType.Opt(DefType.Obj(_, Some(cn))) => cn
+            case DefType.Obj(_, Some(cn)) => cn
+            case DefType.Opt(DefType.Poly(_, Some(cn))) => cn
+            case DefType.Poly(_, Some(cn)) => cn
+            case _ => ""
           }
-          case DefType.Obj(_, Some("lightdb.spatial.Geo.Point")) =>
-            val p = json.as[Geo.Point]
-            add(new LatLonPoint(field.name, p.latitude, p.longitude))
-            add(new StoredField(field.name, JsonFormatter.Compact(p.json)))
-          case DefType.Str => json match {
-            case Null => add(new StringField(field.name, Field.NullString, fs))
-            case _ => add(new StringField(field.name, json.asString, fs))
+          if (className.startsWith("lightdb.spatial.Geo")) {
+            if (json != Null) createGeoFields(className, field, json, add)
+          } else {
+            d match {
+              case DefType.Str => json match {
+                case Null => add(new StringField(field.name, Field.NullString, fs))
+                case _ => add(new StringField(field.name, json.asString, fs))
+              }
+              case DefType.Json | DefType.Obj(_, _) => add(new StringField(field.name, JsonFormatter.Compact(json), fs))
+              case DefType.Opt(d) => addJson(json, d)
+              case DefType.Arr(d) => json.asVector.foreach(json => addJson(json, d))
+              case DefType.Bool => add(new IntField(field.name, if (json.asBoolean) 1 else 0, fs))
+              case DefType.Int => add(new LongField(field.name, json.asLong, fs))
+              case DefType.Dec => add(new DoubleField(field.name, json.asDouble, fs))
+              case _ => throw new UnsupportedOperationException(s"Unsupported definition (field: ${field.name}, className: $className): $d for $json")
+            }
           }
-          case DefType.Json | DefType.Obj(_, _) => add(new StringField(field.name, JsonFormatter.Compact(json), fs))
-          case DefType.Opt(d) => addJson(json, d)
-          case DefType.Arr(d) => json.asVector.foreach(json => addJson(json, d))
-          case DefType.Bool => add(new IntField(field.name, if (json.asBoolean) 1 else 0, fs))
-          case DefType.Int => add(new LongField(field.name, json.asLong, fs))
-          case DefType.Dec => add(new DoubleField(field.name, json.asDouble, fs))
-          case _ => throw new UnsupportedOperationException(s"Unsupported definition (${field.name}): $d for $json")
         }
         addJson(json, field.rw.definition)
 
@@ -148,8 +185,6 @@ class LuceneStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory:
     if (limit <= 0) throw new RuntimeException(s"Limit must be a positive value, but set to $limit")
     def search(total: Option[Int]): TopFieldDocs = {
       val topFieldDocs = indexSearcher.search(q, total.getOrElse(limit), s, query.scoreDocs)
-//      val collectorManager = new TopFieldCollectorManager(s, total.getOrElse(query.offset + limit), null, Int.MaxValue, false)
-//      val topFieldDocs: TopFieldDocs = indexSearcher.search(q, collectorManager)
       val totalHits = total.getOrElse(topFieldDocs.totalHits.value.toInt)
       if (totalHits > topFieldDocs.scoreDocs.length && total.isEmpty && query.limit.forall(l => l + query.offset > limit)) {
         search(Some(query.limit.map(l => math.min(l, totalHits)).getOrElse(totalHits)))
@@ -218,10 +253,10 @@ class LuceneStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory:
         case (json, score) => MaterializedIndex[Doc, Model](json, collection.model).asInstanceOf[V] -> score
       }
       case Conversion.Json(fields) => jsonIterator(fields).asInstanceOf[Iterator[(V, Double)]]
-      case m: Conversion.Distance[Doc] => idsAndScores.iterator.map {
+      case Conversion.Distance(field, from, sort, radius) => idsAndScores.iterator.map {
         case (id, score) =>
           val doc = collection(id)(transaction)
-          val distance = m.field.get(doc).map(d => Spatial.distance(m.from, d))
+          val distance = field.get(doc).map(d => Spatial.distance(from, d))
           DistanceAndDoc(doc, distance) -> score
       }
     }
@@ -259,7 +294,17 @@ class LuceneStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory:
         parser.setSplitOnWhitespace(true)
         parser.parse(query)
       case Filter.Distance(fieldName, from, radius) =>
-        LatLonPoint.newDistanceQuery(fieldName, from.latitude, from.longitude, radius.toMeters)
+        val field = collection.model.fieldByName[Geo](fieldName)
+        val className = field.rw.definition match {
+          case DefType.Opt(DefType.Obj(_, Some(cn))) => cn
+          case DefType.Obj(_, Some(cn)) => cn
+          case _ => ""
+        }
+//        if (className == "lightdb.spatial.Geo.Point") {
+          LatLonPoint.newDistanceQuery(fieldName, from.latitude, from.longitude, radius.toMeters)
+//        } else {
+//          LatLonShape.newDistanceQuery(fieldName, )
+//        }
       case Filter.Multi(minShould, clauses) =>
         val b = new BooleanQuery.Builder
         val hasShould = clauses.exists(c => c.condition == Condition.Should || c.condition == Condition.Filter)