diff --git a/.jvmopts b/.jvmopts new file mode 100644 index 00000000..d342c09b --- /dev/null +++ b/.jvmopts @@ -0,0 +1,7 @@ +-Xms1024M +-Xmx4096M +-Xss512M +-XX:MaxMetaspaceSize=4096M +-XX:ReservedCodeCacheSize=500M +-XX:+TieredCompilation +-XX:-UseGCOverheadLimit \ No newline at end of file diff --git a/README.md b/README.md index aeebc599..0f7bd801 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,21 @@ # lightdb -Prototype database concept using Lucene and HaloDB +Prototype database concept using pluggable store + indexer -First working release with minimal functionality in `0.1.0` +## Provided Stores +- Yahoo's HaloDB (https://github.com/yahoo/HaloDB) - Preferred for performance +- MapDB (https://mapdb.org) +- Facebook's RocksDB (https://rocksdb.org) -## 0.2 TODO -- [X] Support ObjectStore.all to iterate over the entire database +## Provided Indexers +- Apache Lucene (https://lucene.apache.org) - Most featureful +- SQLite (https://www.sqlite.org) - Fastest + +## 1.0 TODO +- [ ] Full implementations for indexers + - [ ] Apache Lucene index types + - [ ] SQLite index types +- [ ] More performance improvements to SQLite integration +- [ ] Better RocksDB performance - [ ] Create backup and restore features - [ ] Real-time backup (write changes to incremental file) - [ ] Complete dump and restore @@ -12,9 +23,4 @@ First working release with minimal functionality in `0.1.0` - [ ] Testing of empty database loads from backups if available - [ ] Data integrity checks - [ ] Verify data identical between store and index - - [ ] Rebuild index from store - -## 1.0 TODO -- [ ] Complete Lucene type support -- [ ] Create benchmark tool to evaluate performance of basic operations to see how well this performs -- [ ] Create an SBT plugin to update base traits for case classes (ex. Person would generate PersonFields trait to be mixed into Person companion) \ No newline at end of file + - [ ] Rebuild index from store \ No newline at end of file diff --git a/build.sbt b/build.sbt index d6d309cb..ec23bcb1 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ // Scala versions val scala213 = "2.13.13" -val scala3 = "3.3.3" +val scala3 = "3.4.1" val scala2 = List(scala213) val allScalaVersions = scala3 :: scala2 @@ -18,7 +18,7 @@ ThisBuild / organization := org ThisBuild / version := "0.4.0-SNAPSHOT" ThisBuild / scalaVersion := scala213 ThisBuild / crossScalaVersions := allScalaVersions -ThisBuild / scalacOptions ++= Seq("-unchecked", "-deprecation") +ThisBuild / scalacOptions ++= Seq("-unchecked", "-deprecation", "-Yno-decode-stacktraces") ThisBuild / javacOptions ++= Seq("-source", "1.8", "-target", "1.8") ThisBuild / publishTo := sonatypePublishTo.value @@ -43,7 +43,7 @@ ThisBuild / outputStrategy := Some(StdoutOutput) ThisBuild / Test / testOptions += Tests.Argument(TestFrameworks.ScalaTest, "-oDF") -val collectionCompatVersion: String = "2.11.0" +val collectionCompatVersion: String = "2.12.0" val haloDBVersion: String = "0.5.6" val rocksDBVersion: String = "9.0.0" val catsEffectVersion: String = "3.5.4" @@ -51,7 +51,7 @@ val fabricVersion: String = "1.14.2" val fs2Version: String = "3.10.2" val scribeVersion: String = "3.13.2" val luceneVersion: String = "9.10.0" -val sqliteVersion: String = "3.45.2.0" +val sqliteVersion: String = "3.45.3.0" val scalaTestVersion: String = "3.2.18" val catsEffectTestingVersion: String = "1.5.0" diff --git a/core/src/main/scala/lightdb/Collection.scala b/core/src/main/scala/lightdb/Collection.scala index efe879a9..66010490 100644 --- a/core/src/main/scala/lightdb/Collection.scala +++ b/core/src/main/scala/lightdb/Collection.scala @@ -85,7 +85,7 @@ abstract class Collection[D <: Document[D]](val collectionName: String, name = name, createKey = createKey, createV = createV, - store = db.createStoreInternal(s"$collectionName.indexed.$name"), + loadStore = () => db.createStoreInternal(s"$collectionName.indexed.$name"), collection = this, maxLinks = maxLinks ) diff --git a/core/src/main/scala/lightdb/IndexedLink.scala b/core/src/main/scala/lightdb/IndexedLink.scala index 1dae7aab..40b99995 100644 --- a/core/src/main/scala/lightdb/IndexedLink.scala +++ b/core/src/main/scala/lightdb/IndexedLink.scala @@ -1,6 +1,6 @@ package lightdb -import fabric.rw.RW +import fabric.rw._ case class IndexedLink[D <: Document[D]](_id: Id[IndexedLink[D]], links: List[Id[D]]) extends Document[IndexedLink[D]] diff --git a/core/src/main/scala/lightdb/IndexedLinks.scala b/core/src/main/scala/lightdb/IndexedLinks.scala index 17f2c07a..41dad761 100644 --- a/core/src/main/scala/lightdb/IndexedLinks.scala +++ b/core/src/main/scala/lightdb/IndexedLinks.scala @@ -5,9 +5,11 @@ import cats.effect.IO case class IndexedLinks[V, D <: Document[D]](name: String, createKey: V => String, createV: D => V, - store: Store, + loadStore: () => Store, collection: Collection[D], maxLinks: MaxLinks) { + lazy val store: Store = loadStore() + protected[lightdb] def add(doc: D): IO[Unit] = { val v = createV(doc) for { diff --git a/core/src/main/scala/lightdb/KeyValue.scala b/core/src/main/scala/lightdb/KeyValue.scala index 10a5d29a..2b890a80 100644 --- a/core/src/main/scala/lightdb/KeyValue.scala +++ b/core/src/main/scala/lightdb/KeyValue.scala @@ -1,7 +1,7 @@ package lightdb import fabric.Json -import fabric.rw.RW +import fabric.rw._ case class KeyValue(_id: Id[KeyValue], value: Json) extends Document[KeyValue] diff --git a/core/src/main/scala/lightdb/LightDB.scala b/core/src/main/scala/lightdb/LightDB.scala index ce05bf8d..c71c1869 100644 --- a/core/src/main/scala/lightdb/LightDB.scala +++ b/core/src/main/scala/lightdb/LightDB.scala @@ -2,7 +2,7 @@ package lightdb import cats.effect.IO import cats.implicits.{catsSyntaxApplicativeByName, catsSyntaxParallelSequence1} -import fabric.rw.RW +import fabric.rw._ import lightdb.upgrade.DatabaseUpgrade import java.nio.file.Path diff --git a/project/plugins.sbt b/project/plugins.sbt index d68f5c25..5413432d 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,15 +1,15 @@ addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "1.3.2") addSbtPlugin("org.portable-scala" % "sbt-scala-native-crossproject" % "1.3.2") addSbtPlugin("org.scala-js" % "sbt-scalajs" % "1.16.0") -addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.4.17") +addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.5.1") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.13") +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.10.0") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0") -addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.3") +addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.7") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.0") +addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.11") -addSbtPlugin("com.github.sbt" % "sbt-git" % "2.0.0") +addSbtPlugin("com.github.sbt" % "sbt-git" % "2.0.1") addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.4") \ No newline at end of file