Replaced SpatiaLite with custom function and cleaned up spatial support

core/src/main/scala/lightdb/spatial/Geo.scala

@@ -1,13 +1,16 @@
 package lightdb.spatial
 
+import fabric._
 import fabric.rw._
 
 sealed trait Geo {
   def center: Geo.Point
 }
 
 object Geo {
-  implicit val pRW: RW[Point] = RW.gen
+  implicit val pRW: RW[Point] = RW.gen[Point]
+    .withPreWrite(_.merge(obj("type" -> "Point")))
+    .withPostRead((_, json) => json.merge(obj("type" -> "Point")))
   private implicit val mpRW: RW[MultiPoint] = RW.gen
   private implicit val lsRW: RW[Line] = RW.gen
   private implicit val mlsRW: RW[MultiLine] = RW.gen

core/src/main/scala/lightdb/spatial/Spatial.scala

@@ -8,7 +8,7 @@ object Spatial {
   private lazy val context = SpatialContext.GEO
 
   def distance(p1: Geo, p2: Geo): Distance = {
-    val point1 = context.getShapeFactory.pointLatLon(p1.center.latitude, p2.center.longitude)
+    val point1 = context.getShapeFactory.pointLatLon(p1.center.latitude, p1.center.longitude)
     val point2 = context.getShapeFactory.pointLatLon(p2.center.latitude, p2.center.longitude)
     val degrees = context.calcDistance(point1, point2)
     val distance = DistanceUtils.degrees2Dist(degrees, DistanceUtils.EARTH_MEAN_RADIUS_KM)

core/src/test/scala/spec/AbstractSpatialSpec.scala

@@ -85,11 +85,15 @@ abstract class AbstractSpatialSpec extends AnyWordSpec with Matchers { spec =>
         ).iterator.toList
         val people = list.map(_.doc)
         val distances = list.map(_.distance.get.mi)
-        people.map(_.name) should be(List("Jane Doe", "John Doe"))
-        distances should (be (List(28.307644231281916, 356.7163915969269)) or be(List(28.307644231281916, 356.7163915969269)))
+        people.zip(distances).map {
+          case (p, d) => p.name -> d
+        } should be(List(
+          "Jane Doe" -> 28.55539552714398,
+          "John Doe" -> 1316.1301092705082
+        ))
       }
     }
-    "sort by distance from Noble using altPoint" in {
+    "sort by distance from Noble using geo" in {
       DB.people.transaction { implicit transaction =>
         val list = DB.people.query.search.distance(
           _.geo,
@@ -98,8 +102,12 @@ abstract class AbstractSpatialSpec extends AnyWordSpec with Matchers { spec =>
         ).iterator.toList
         val people = list.map(_.doc)
         val distances = list.map(_.distance.get.mi)
-        people.map(_.name) should be(List("Jane Doe", "Bob Dole"))
-        distances should (be (List(15.489309276333513, 460.868070665109)) or be(List(28.307644231281916, 460.868070665109)))
+        people.zip(distances).map {
+          case (p, d) => p.name -> d
+        } should be(List(
+          "Jane Doe" -> 16.01508397712445,
+          "Bob Dole" -> 695.6419047674393
+        ))
       }
     }
     "truncate the database" in {

lucene/src/main/scala/lightdb/lucene/LuceneStore.scala

@@ -122,10 +122,9 @@ class LuceneStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory:
             val sorted = new SortedDocValuesField(fieldSortName, bytes)
             add(sorted)
           case NumInt(l, _) => add(new NumericDocValuesField(fieldSortName, l))
-          case obj: Obj if obj.reference.nonEmpty => obj.reference.get match {
-            case Geo.Point(latitude, longitude) => add(new LatLonDocValuesField(fieldSortName, latitude, longitude))
-            case _ => // Ignore
-          }
+          case j if field.isSpatial && j != Null =>
+            val g = j.as[Geo]
+            add(new LatLonDocValuesField(fieldSortName, g.center.latitude, g.center.longitude))
           case _ => // Ignore
         }
         fields

sql/src/main/scala/lightdb/sql/SQLArg.scala

@@ -33,7 +33,7 @@ object SQLArg {
         case d: Double => ps.setDouble(index, d)
         case bd: BigDecimal => ps.setDouble(index, bd.toDouble)
         case json: Json => ps.setString(index, JsonFormatter.Compact(json))
-        case point: Geo.Point => ps.setString(index, s"POINT(${point.longitude} ${point.latitude})")
+//        case point: Geo.Point => ps.setString(index, s"POINT(${point.longitude} ${point.latitude})")
         case _ =>
           val json = if (field.rw.definition.isOpt) {
             Some(value).asInstanceOf[F].json(field.rw)

sql/src/main/scala/lightdb/sql/SQLStore.scala

@@ -276,7 +276,7 @@ abstract class SQLStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]] exten
           jsonFromFields(fields).asInstanceOf[V]
         case Conversion.Distance(field, _, _, _) =>
           val fieldName = s"${field.name}Distance"
-          val distance = Option(rs.getObject(fieldName)).map(_.asInstanceOf[java.lang.Double].doubleValue()).map(Distance.apply)
+          val distance = Option(rs.getObject(fieldName)).map(_.toString.toDouble).map(Distance.apply)
           val doc = getDoc(rs)
           DistanceAndDoc(doc, distance).asInstanceOf[V]
       }

sqlite/src/main/scala/lightdb/sql/SQLiteStore.scala

@@ -2,12 +2,14 @@ package lightdb.sql
 
 import fabric._
 import fabric.define.DefType
+import fabric.io.{JsonFormatter, JsonParser}
 import fabric.rw._
 import lightdb.collection.Collection
 import lightdb.sql.connect.{ConnectionManager, DBCPConnectionManager, SQLConfig, SingleConnectionManager}
 import lightdb.{Field, LightDB}
 import lightdb.doc.{Document, DocumentModel}
 import lightdb.filter.Filter
+import lightdb.spatial.{Geo, Spatial}
 import lightdb.store.{Conversion, Store, StoreManager, StoreMode}
 import lightdb.transaction.Transaction
 import org.sqlite.{SQLiteConfig, SQLiteOpenMode}
@@ -21,18 +23,29 @@ import java.util.regex.Pattern
 class SQLiteStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](val connectionManager: ConnectionManager,
                                                                      val connectionShared: Boolean,
                                                                      val storeMode: StoreMode) extends SQLStore[Doc, Model] {
-  private val PointRegex = """POINT\((.+) (.+)\)""".r
-  private val OptPointRegex = """\[POINT\((.+) (.+)\)\]""".r
-
   override protected def initTransaction()(implicit transaction: Transaction[Doc]): Unit = {
     val c = connectionManager.getConnection
     if (hasSpatial) {
       scribe.info(s"${collection.name} has spatial features. Enabling...")
-      val s = c.createStatement()
-      s.executeUpdate(s"SELECT load_extension('${SQLiteStore.spatialitePath}');")
-      val hasGeometryColumns = this.tables(c).contains("geometry_columns")
-      if (!hasGeometryColumns) s.executeUpdate("SELECT InitSpatialMetaData()")
-      s.close()
+      org.sqlite.Function.create(c, "DISTANCE", new org.sqlite.Function() {
+        override def xFunc(): Unit = {
+          def s(index: Int): Option[Geo] = Option(value_text(index))
+            .map(s => JsonParser(s).as[Geo])
+          val shape1 = s(0)
+          val shape2 = s(1)
+          val distance = (shape1, shape2) match {
+            case (Some(s1), Some(s2)) =>
+              Some(Spatial.distance(s1, s2))
+            case _ => None
+          }
+          distance match {
+            case Some(d) =>
+              val meters = d.valueInMeters
+              result(meters)
+            case None => result()
+          }
+        }
+      })
     }
     org.sqlite.Function.create(c, "REGEXP", new org.sqlite.Function() {
       override def xFunc(): Unit = {
@@ -47,77 +60,15 @@ class SQLiteStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](val connect
 
   override protected def tables(connection: Connection): Set[String] = SQLiteStore.tables(connection)
 
-  override protected def toJson(value: Any, rw: RW[_]): Json = {
-    val className = rw.definition match {
-      case DefType.Opt(d) => d.className
-      case d => d.className
-    }
-    if (value != null && className.contains("lightdb.spatial.GeoPoint")) {
-      value.toString match {
-        case PointRegex(longitude, latitude) => obj(
-          "latitude" -> num(latitude.toDouble),
-          "longitude" -> num(longitude.toDouble)
-        )
-        case OptPointRegex(longitude, latitude) => obj(
-          "latitude" -> num(latitude.toDouble),
-          "longitude" -> num(longitude.toDouble)
-        )
-      }
-    } else {
-      super.toJson(value, rw)
-    }
-  }
-
-  override protected def field2Value(field: Field[Doc, _]): String = {
-    val className = field.rw.definition match {
-      case DefType.Opt(d) => d.className
-      case d => d.className
-    }
-    if (className.contains("lightdb.spatial.GeoPoint")) {
-      "GeomFromText(?, 4326)"
-    } else {
-      super.field2Value(field)
-    }
-  }
-
-  override protected def fieldPart[V](field: Field[Doc, V]): SQLPart = {
-    val className = field.rw.definition match {
-      case DefType.Opt(d) => d.className
-      case d => d.className
-    }
-    if (className.contains("lightdb.spatial.GeoPoint")) {
-      SQLPart(s"AsText(${field.name}) AS ${field.name}")
-    } else {
-      super.fieldPart(field)
-    }
-  }
-
-  override protected def extraFieldsForDistance(d: Conversion.Distance[Doc, _]): List[SQLPart] = {
-    List(SQLPart(s"ST_Distance(GeomFromText('POINT(${d.from.longitude} ${d.from.latitude})', 4326), ${d.field.name}, true) AS ${d.field.name}Distance"))
-  }
+  override protected def extraFieldsForDistance(d: Conversion.Distance[Doc, _]): List[SQLPart] =
+    List(SQLPart(s"DISTANCE(${d.field.name}, ?) AS ${d.field.name}Distance", List(SQLArg.JsonArg(d.from.json))))
 
   override protected def distanceFilter(f: Filter.Distance[Doc]): SQLPart =
-    SQLPart(s"ST_Distance(${f.fieldName}, GeomFromText(?, 4326), true) <= ?", List(SQLArg.GeoPointArg(f.from), SQLArg.DoubleArg(f.radius.m)))
-
-  override protected def addColumn(field: Field[Doc, _])(implicit transaction: Transaction[Doc]): Unit = {
-    if (field.rw.definition.className.contains("lightdb.spatial.GeoPoint")) {
-      executeUpdate(s"SELECT AddGeometryColumn('${collection.name}', '${field.name}', 4326, 'POINT', 'XY');")
-    } else {
-      super.addColumn(field)
-    }
-  }
+    SQLPart(s"${f.fieldName}Distance <= ?", List(SQLArg.DoubleArg(f.radius.valueInMeters)))
+//    SQLPart(s"${f.fieldName} DISTANCE ? <= ?", List(SQLArg.JsonArg(f.from.json), SQLArg.DoubleArg(f.radius.m)))
 }
 
 object SQLiteStore extends StoreManager {
-  private lazy val spatialitePath: String = {
-    val file = Files.createTempFile("mod_spatialite", ".so")
-    val input = getClass.getClassLoader.getResourceAsStream("mod_spatialite.so")
-    Files.copy(input, file, StandardCopyOption.REPLACE_EXISTING)
-    file.toAbsolutePath.toString match {
-      case s => s.substring(0, s.length - 3)
-    }
-  }
-
   def singleConnectionManager(file: Option[Path]): ConnectionManager = {
     val connection: Connection = {
       val path = file match {

sqlite/src/test/scala/spec/SQLiteSpatialSpec.scala

@@ -1,8 +1,9 @@
 package spec
 
-import lightdb.sql.SQLiteStore
+import lightdb.sql.{SQLQueryBuilder, SQLiteStore}
 import lightdb.store.StoreManager
 
+@EmbeddedTest
 class SQLiteSpatialSpec extends AbstractSpatialSpec {
   override protected def storeManager: StoreManager = SQLiteStore
 }