Fixes to RocksDB support and addition of KeyValueSpec

all/src/test/scala/spec/RocksDBAndLuceneSpec.scala

@@ -0,0 +1,13 @@
+package spec
+
+import lightdb.lucene.LuceneStore
+import lightdb.rocksdb.RocksDBStore
+import lightdb.store.{StoreManager, StoreMode}
+import lightdb.store.split.SplitStoreManager
+
+@EmbeddedTest
+class RocksDBAndLuceneSpec extends AbstractBasicSpec {
+  override protected def filterBuilderSupported: Boolean = true
+
+  override def storeManager: StoreManager = SplitStoreManager(RocksDBStore, LuceneStore, searchingMode = StoreMode.Indexes)
+}

core/src/test/scala/spec/AbstractBasicSpec.scala

@@ -508,36 +508,6 @@ abstract class AbstractBasicSpec extends AnyWordSpec with Matchers { spec =>
           }
       }
     }
-
-    /*override protected def adding(doc: Person)(implicit transaction: Transaction[Person]): Unit = {
-              if (doc.age == 30) {
-                scribe.info(s"Adding: $doc")
-              }
-              db.ageLinks.t.modify(AgeLinks.id(doc.age)) {
-                case Some(links) =>
-                  val result = Some(links.copy(people = (doc._id :: links.people).distinct))
-                  if (doc.age == 30) {
-                    scribe.info(s"Current: $links, adding: $doc, Result: $result")
-                  }
-                  result
-                case None =>
-                  if (doc.age == 30) {
-                    scribe.info(s"New AgeLink: $doc")
-                  }
-                  Some(AgeLinks(doc.age, List(doc._id)))
-              }
-            }
-            override protected def modifying(oldDoc: Person, newDoc: Person)(implicit transaction: Transaction[Person]): Unit = adding(newDoc)
-            override protected def removing(doc: Person)(implicit transaction: Transaction[Person]): Unit = db.ageLinks.t.modify(AgeLinks.id(doc.age)) {
-              case Some(links) =>
-                val l = links.copy(people = links.people.filterNot(_ == doc._id))
-                if (l.people.isEmpty) {
-                  None
-                } else {
-                  Some(l)
-                }
-              case None => None
-            }*/
   }
 
   object InitialSetupUpgrade extends DatabaseUpgrade {

core/src/test/scala/spec/AbstractKeyValueSpec.scala

@@ -0,0 +1,161 @@
+package spec
+
+import fabric.rw.RW
+import lightdb.collection.Collection
+import lightdb.{Id, LightDB}
+import lightdb.doc.{Document, DocumentModel, JsonConversion}
+import lightdb.store.StoreManager
+import lightdb.upgrade.DatabaseUpgrade
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+
+import java.nio.file.Path
+
+abstract class AbstractKeyValueSpec extends AnyWordSpec with Matchers { spec =>
+  val CreateRecords = 100_000
+
+  private val adam = User("Adam", 21, _id = User.id("adam"))
+  private val brenda = User("Brenda", 11, _id = User.id("brenda"))
+  private val charlie = User("Charlie", 35, _id = User.id("charlie"))
+  private val diana = User("Diana", 15, _id = User.id("diana"))
+  private val evan = User("Evan", 53, _id = User.id("evan"))
+  private val fiona = User("Fiona", 23, _id = User.id("fiona"))
+  private val greg = User("Greg", 12, _id = User.id("greg"))
+  private val hanna = User("Hanna", 62, _id = User.id("hanna"))
+  private val ian = User("Ian", 89, _id = User.id("ian"))
+  private val jenna = User("Jenna", 4, _id = User.id("jenna"))
+  private val kevin = User("Kevin", 33, _id = User.id("kevin"))
+  private val linda = User("Linda", 72, _id = User.id("linda"))
+  private val mike = User("Mike", 42, _id = User.id("mike"))
+  private val nancy = User("Nancy", 22, _id = User.id("nancy"))
+  private val oscar = User("Oscar", 21, _id = User.id("oscar"))
+  private val penny = User("Penny", 2, _id = User.id("penny"))
+  private val quintin = User("Quintin", 99, _id = User.id("quintin"))
+  private val ruth = User("Ruth", 102, _id = User.id("ruth"))
+  private val sam = User("Sam", 81, _id = User.id("sam"))
+  private val tori = User("Tori", 30, _id = User.id("tori"))
+  private val uba = User("Uba", 21, _id = User.id("uba"))
+  private val veronica = User("Veronica", 13, _id = User.id("veronica"))
+  private val wyatt = User("Wyatt", 30, _id = User.id("wyatt"))
+  private val xena = User("Xena", 63, _id = User.id("xena"))
+  private val yuri = User("Yuri", 30, _id = User.id("yuri"))
+  private val zoey = User("Zoey", 101, _id = User.id("zoey"))
+
+  private val names = List(
+    adam, brenda, charlie, diana, evan, fiona, greg, hanna, ian, jenna, kevin, linda, mike, nancy, oscar, penny,
+    quintin, ruth, sam, tori, uba, veronica, wyatt, xena, yuri, zoey
+  )
+
+  protected lazy val specName: String = getClass.getSimpleName
+
+  protected var db: DB = new DB
+
+  specName should {
+    "initialize the database" in {
+      db.init() should be(true)
+    }
+    "verify the database is empty" in {
+      db.users.transaction { implicit transaction =>
+        db.users.count should be(0)
+      }
+    }
+    "insert the records" in {
+      db.users.transaction { implicit transaction =>
+        db.users.insert(names) should not be None
+      }
+    }
+    "retrieve the first record by _id -> id" in {
+      db.users.transaction { implicit transaction =>
+        db.users(_._id -> adam._id) should be(adam)
+      }
+    }
+    "retrieve the first record by id" in {
+      db.users.transaction { implicit transaction =>
+        db.users(adam._id) should be(adam)
+      }
+    }
+    "count the records in the database" in {
+      db.users.transaction { implicit transaction =>
+        db.users.count should be(26)
+      }
+    }
+    "stream the records in the database" in {
+      db.users.transaction { implicit transaction =>
+        val ages = db.users.iterator.map(_.age).toSet
+        ages should be(Set(101, 42, 89, 102, 53, 13, 2, 22, 12, 81, 35, 63, 99, 23, 30, 4, 21, 33, 11, 72, 15, 62))
+      }
+    }
+    "delete some records" in {
+      db.users.transaction { implicit transaction =>
+        db.users.delete(_._id -> linda._id) should be(true)
+        db.users.delete(_._id -> yuri._id) should be(true)
+      }
+    }
+    "verify the records were deleted" in {
+      db.users.transaction { implicit transaction =>
+        db.users.count should be(24)
+      }
+    }
+    "modify a record" in {
+      db.users.transaction { implicit transaction =>
+        db.users.modify(adam._id) {
+          case Some(p) => Some(p.copy(name = "Allan"))
+          case None => fail("Adam was not found!")
+        }
+      } match {
+        case Some(p) => p.name should be("Allan")
+        case None => fail("Allan was not returned!")
+      }
+    }
+    "verify the record has been renamed" in {
+      db.users.transaction { implicit transaction =>
+        db.users(_._id -> adam._id).name should be("Allan")
+      }
+    }
+    "insert a lot more names" in {
+      db.users.transaction { implicit transaction =>
+        val p = (1 to CreateRecords).toList.map { index =>
+          User(
+            name = s"Unique Snowflake $index",
+            age = if (index > 100) 0 else index,
+          )
+        }
+        db.users.insert(p)
+      }
+    }
+    "verify the correct number of people exist in the database" in {
+      db.users.transaction { implicit transaction =>
+        db.users.count should be(CreateRecords + 24)
+      }
+    }
+    "truncate the collection again" in {
+      db.users.transaction { implicit transaction =>
+        db.users.truncate() should be(CreateRecords + 24)
+      }
+    }
+    "dispose the database" in {
+      db.dispose()
+    }
+  }
+
+  def storeManager: StoreManager
+
+  class DB extends LightDB {
+    lazy val directory: Option[Path] = Some(Path.of(s"db/$specName"))
+
+    val users: Collection[User, User.type] = collection(User)
+
+    override def storeManager: StoreManager = spec.storeManager
+
+    override def upgrades: List[DatabaseUpgrade] = Nil
+  }
+
+  case class User(name: String, age: Int, _id: Id[User] = User.id()) extends Document[User]
+
+  object User extends DocumentModel[User] with JsonConversion[User] {
+    override implicit val rw: RW[User] = RW.gen
+
+    val name: I[String] = field.index("name", _.name)
+    val age: F[Int] = field("age", _.age)
+  }
+}

halodb/src/test/scala/spec/HaloDBSpec.scala

@@ -0,0 +1,9 @@
+package spec
+
+import lightdb.halodb.HaloDBStore
+import lightdb.store.StoreManager
+
+@EmbeddedTest
+class HaloDBSpec extends AbstractKeyValueSpec {
+  override def storeManager: StoreManager = HaloDBStore
+}

rocksdb/src/main/scala/lightdb/rocksdb/RocksDBStore.scala

@@ -8,16 +8,18 @@ import lightdb._
 import lightdb.field.Field._
 import lightdb.doc.{Document, DocumentModel}
 import lightdb.materialized.MaterializedAggregate
-import lightdb.store.{Conversion, Store, StoreMode}
+import lightdb.store.{Conversion, Store, StoreManager, StoreMode}
 import lightdb.transaction.Transaction
-import org.rocksdb.{FlushOptions, RocksDB, RocksIterator}
+import org.rocksdb.{FlushOptions, Options, RocksDB, RocksIterator}
 
 import java.nio.file.{Files, Path}
 
 class RocksDBStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory: Path, val storeMode: StoreMode) extends Store[Doc, Model] {
   private lazy val db: RocksDB = {
     Files.createDirectories(directory.getParent)
-    RocksDB.open(directory.toAbsolutePath.toString)
+    val options = new Options()
+      .setCreateIfMissing(true)
+    RocksDB.open(options, directory.toAbsolutePath.toString)
   }
 
   override def init(collection: Collection[Doc, Model]): Unit = {
@@ -85,7 +87,7 @@ class RocksDBStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory:
     db.close()
   }
 
-  private def iterator(rocksIterator: RocksIterator, value: Boolean = true): Iterator[Array[Byte]] = new Iterator[Array[Byte]] {
+  private def iteratorOld(rocksIterator: RocksIterator, value: Boolean = true): Iterator[Array[Byte]] = new Iterator[Array[Byte]] {
     override def hasNext: Boolean = rocksIterator.isValid
 
     override def next(): Array[Byte] = try {
@@ -98,4 +100,32 @@ class RocksDBStore[Doc <: Document[Doc], Model <: DocumentModel[Doc]](directory:
       rocksIterator.next()
     }
   }
+
+  private def iterator(rocksIterator: RocksIterator, value: Boolean = true): Iterator[Array[Byte]] = new Iterator[Array[Byte]] {
+    // Initialize the iterator to the first position
+    rocksIterator.seekToFirst()
+
+    override def hasNext: Boolean = rocksIterator.isValid
+
+    override def next(): Array[Byte] = {
+      if (!hasNext) throw new NoSuchElementException("No more elements in the RocksDB iterator")
+
+      val result = if (value) {
+        rocksIterator.value()
+      } else {
+        rocksIterator.key()
+      }
+
+      // Move to the next entry after retrieving the current value or key
+      rocksIterator.next()
+      result
+    }
+  }
+}
+
+object RocksDBStore extends StoreManager {
+  override def create[Doc <: Document[Doc], Model <: DocumentModel[Doc]](db: LightDB,
+                                                                         name: String,
+                                                                         storeMode: StoreMode): Store[Doc, Model] =
+    new RocksDBStore[Doc, Model](db.directory.get.resolve(name), storeMode)
 }

rocksdb/src/test/scala/spec/RocksDBSpec.scala

@@ -0,0 +1,9 @@
+package spec
+
+import lightdb.rocksdb.RocksDBStore
+import lightdb.store.StoreManager
+
+@EmbeddedTest
+class RocksDBSpec extends AbstractKeyValueSpec {
+  override def storeManager: StoreManager = RocksDBStore
+}