diff --git a/.jvmopts b/.jvmopts new file mode 100644 index 00000000..d342c09b --- /dev/null +++ b/.jvmopts @@ -0,0 +1,7 @@ +-Xms1024M +-Xmx4096M +-Xss512M +-XX:MaxMetaspaceSize=4096M +-XX:ReservedCodeCacheSize=500M +-XX:+TieredCompilation +-XX:-UseGCOverheadLimit \ No newline at end of file diff --git a/README.md b/README.md index aeebc599..0f7bd801 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,21 @@ # lightdb -Prototype database concept using Lucene and HaloDB +Prototype database concept using pluggable store + indexer -First working release with minimal functionality in `0.1.0` +## Provided Stores +- Yahoo's HaloDB (https://github.com/yahoo/HaloDB) - Preferred for performance +- MapDB (https://mapdb.org) +- Facebook's RocksDB (https://rocksdb.org) -## 0.2 TODO -- [X] Support ObjectStore.all to iterate over the entire database +## Provided Indexers +- Apache Lucene (https://lucene.apache.org) - Most featureful +- SQLite (https://www.sqlite.org) - Fastest + +## 1.0 TODO +- [ ] Full implementations for indexers + - [ ] Apache Lucene index types + - [ ] SQLite index types +- [ ] More performance improvements to SQLite integration +- [ ] Better RocksDB performance - [ ] Create backup and restore features - [ ] Real-time backup (write changes to incremental file) - [ ] Complete dump and restore @@ -12,9 +23,4 @@ First working release with minimal functionality in `0.1.0` - [ ] Testing of empty database loads from backups if available - [ ] Data integrity checks - [ ] Verify data identical between store and index - - [ ] Rebuild index from store - -## 1.0 TODO -- [ ] Complete Lucene type support -- [ ] Create benchmark tool to evaluate performance of basic operations to see how well this performs -- [ ] Create an SBT plugin to update base traits for case classes (ex. Person would generate PersonFields trait to be mixed into Person companion) \ No newline at end of file + - [ ] Rebuild index from store \ No newline at end of file diff --git a/build.sbt b/build.sbt index 1a7a509d..42ea2730 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ // Scala versions val scala213 = "2.13.13" -val scala3 = "3.3.3" +val scala3 = "3.4.1" val scala2 = List(scala213) val allScalaVersions = scala3 :: scala2 @@ -18,7 +18,7 @@ ThisBuild / organization := org ThisBuild / version := "0.4.0-SNAPSHOT" ThisBuild / scalaVersion := scala213 ThisBuild / crossScalaVersions := allScalaVersions -ThisBuild / scalacOptions ++= Seq("-unchecked", "-deprecation") +ThisBuild / scalacOptions ++= Seq("-unchecked", "-deprecation", "-Yno-decode-stacktraces") ThisBuild / javacOptions ++= Seq("-source", "1.8", "-target", "1.8") ThisBuild / publishTo := sonatypePublishTo.value diff --git a/core/src/main/scala/lightdb/Collection.scala b/core/src/main/scala/lightdb/Collection.scala index efe879a9..66010490 100644 --- a/core/src/main/scala/lightdb/Collection.scala +++ b/core/src/main/scala/lightdb/Collection.scala @@ -85,7 +85,7 @@ abstract class Collection[D <: Document[D]](val collectionName: String, name = name, createKey = createKey, createV = createV, - store = db.createStoreInternal(s"$collectionName.indexed.$name"), + loadStore = () => db.createStoreInternal(s"$collectionName.indexed.$name"), collection = this, maxLinks = maxLinks ) diff --git a/core/src/main/scala/lightdb/IndexedLink.scala b/core/src/main/scala/lightdb/IndexedLink.scala index 1dae7aab..40b99995 100644 --- a/core/src/main/scala/lightdb/IndexedLink.scala +++ b/core/src/main/scala/lightdb/IndexedLink.scala @@ -1,6 +1,6 @@ package lightdb -import fabric.rw.RW +import fabric.rw._ case class IndexedLink[D <: Document[D]](_id: Id[IndexedLink[D]], links: List[Id[D]]) extends Document[IndexedLink[D]] diff --git a/core/src/main/scala/lightdb/IndexedLinks.scala b/core/src/main/scala/lightdb/IndexedLinks.scala index 17f2c07a..41dad761 100644 --- a/core/src/main/scala/lightdb/IndexedLinks.scala +++ b/core/src/main/scala/lightdb/IndexedLinks.scala @@ -5,9 +5,11 @@ import cats.effect.IO case class IndexedLinks[V, D <: Document[D]](name: String, createKey: V => String, createV: D => V, - store: Store, + loadStore: () => Store, collection: Collection[D], maxLinks: MaxLinks) { + lazy val store: Store = loadStore() + protected[lightdb] def add(doc: D): IO[Unit] = { val v = createV(doc) for { diff --git a/core/src/main/scala/lightdb/KeyValue.scala b/core/src/main/scala/lightdb/KeyValue.scala index 10a5d29a..2b890a80 100644 --- a/core/src/main/scala/lightdb/KeyValue.scala +++ b/core/src/main/scala/lightdb/KeyValue.scala @@ -1,7 +1,7 @@ package lightdb import fabric.Json -import fabric.rw.RW +import fabric.rw._ case class KeyValue(_id: Id[KeyValue], value: Json) extends Document[KeyValue] diff --git a/core/src/main/scala/lightdb/LightDB.scala b/core/src/main/scala/lightdb/LightDB.scala index ce05bf8d..c71c1869 100644 --- a/core/src/main/scala/lightdb/LightDB.scala +++ b/core/src/main/scala/lightdb/LightDB.scala @@ -2,7 +2,7 @@ package lightdb import cats.effect.IO import cats.implicits.{catsSyntaxApplicativeByName, catsSyntaxParallelSequence1} -import fabric.rw.RW +import fabric.rw._ import lightdb.upgrade.DatabaseUpgrade import java.nio.file.Path