Updates

outr · Apr 7, 2024 · a016816 · a016816
1 parent caa86b4
commit a016816
Show file tree

Hide file tree

Showing 4 changed files with 44 additions and 9 deletions.
diff --git a/benchmark/src/main/scala/benchmark/IMDBBenchmark.scala b/benchmark/src/main/scala/benchmark/IMDBBenchmark.scala
@@ -77,11 +77,11 @@ object IMDBBenchmark { // extends IOApp {
     val io = for {
       _ <- implementation.init()
       _ = scribe.info("--- Stage 1 ---")
-      akasFile <- downloadFile(new File(baseDirectory, "title.akas.tsv"), Limit.OneMillion).elapsed
+      akasFile <- downloadFile(new File(baseDirectory, "title.akas.tsv"), Limit.Unlimited).elapsed
       _ = scribe.info("--- Stage 2 ---")
       totalAka <- process(akasFile.value, implementation.map2TitleAka, implementation.persistTitleAka).elapsed
       _ = scribe.info("--- Stage 3 ---")
-      basicsFile <- downloadFile(new File(baseDirectory, "title.basics.tsv"), Limit.OneMillion).elapsed
+      basicsFile <- downloadFile(new File(baseDirectory, "title.basics.tsv"), Limit.Unlimited).elapsed
       _ = scribe.info("--- Stage 4 ---")
       totalBasics <- process(basicsFile.value, implementation.map2TitleBasics, implementation.persistTitleBasics).elapsed
       _ = scribe.info("--- Stage 5 ---")

diff --git a/benchmark/src/main/scala/benchmark/LightDBImplementation.scala b/benchmark/src/main/scala/benchmark/LightDBImplementation.scala
@@ -3,7 +3,7 @@ package benchmark
 import cats.effect.IO
 import fabric.rw.RW
 import lightdb.upgrade.DatabaseUpgrade
-import lightdb.{Collection, Document, Id, IndexedLinks, LightDB}
+import lightdb.{Collection, Document, Id, IndexedLinks, LightDB, MaxLinks}
 
 import java.nio.file.Paths
 
@@ -90,7 +90,7 @@ object LightDBImplementation extends BenchmarkImplementation {
   object TitleAkaLDB extends Collection[TitleAkaLDB]("titleAka", DB) {
     override implicit val rw: RW[TitleAkaLDB] = RW.gen
 
-    val titleId: IndexedLinks[String, TitleAkaLDB] = indexedLinks[String]("titleId", identity, _.titleId)
+    val titleId: IndexedLinks[String, TitleAkaLDB] = indexedLinks[String]("titleId", identity, _.titleId, MaxLinks.OverflowTrim(100))
   }
 
   case class TitleBasicsLDB(tconst: String, titleType: String, primaryTitle: String, originalTitle: String, isAdult: Boolean, startYear: Int, endYear: Int, runtimeMinutes: Int, genres: List[String], _id: Id[TitleBasics]) extends Document[TitleBasics]

diff --git a/core/src/main/scala/lightdb/Collection.scala b/core/src/main/scala/lightdb/Collection.scala
@@ -68,12 +68,15 @@ abstract class Collection[D <: Document[D]](val collectionName: String, db: Ligh
 
   def indexedLinks[V](name: String,
                       createKey: V => String,
-                      createV: D => V): IndexedLinks[V, D] = {
+                      createV: D => V,
+                      maxLinks: MaxLinks = MaxLinks.OverflowTrim()): IndexedLinks[V, D] = {
     val il = IndexedLinks[V, D](
+      name = name,
       createKey = createKey,
       createV = createV,
       store = db.createStore(s"$collectionName.indexed.$name"),
-      this
+      collection = this,
+      maxLinks = maxLinks
     )
     synchronized {
       _indexedLinks = il :: _indexedLinks

diff --git a/core/src/main/scala/lightdb/IndexedLinks.scala b/core/src/main/scala/lightdb/IndexedLinks.scala
@@ -2,16 +2,39 @@ package lightdb
 
 import cats.effect.IO
 
-case class IndexedLinks[V, D <: Document[D]](createKey: V => String,
+case class IndexedLinks[V, D <: Document[D]](name: String,
+                                             createKey: V => String,
                                              createV: D => V,
                                              store: Store,
-                                             collection: Collection[D]) {
+                                             collection: Collection[D],
+                                             maxLinks: MaxLinks) {
   protected[lightdb] def add(doc: D): IO[Unit] = {
     val v = createV(doc)
     for {
       existing <- link(v)
       updated = existing match {
-        case Some(l) => l.copy(links = l.links ::: List(doc._id))
+        case Some(l) =>
+          val links = l.links ::: List(doc._id)
+          val count = links.length
+          val updatedLinks = maxLinks match {
+            case MaxLinks.NoMax => links
+            case MaxLinks.OverflowError(max) => if (count > max) {
+              throw new RuntimeException(s"Link overflow for $name ($max)")
+            } else {
+              links
+            }
+            case MaxLinks.OverflowWarn(max) =>
+              if (count > max) {
+                scribe.warn(s"Link overflow for $name (max: $max, count: $count)")
+              }
+              links
+            case MaxLinks.OverflowTrim(max) => if (count > max) {
+              links.drop(count - max)
+            } else {
+              links
+            }
+          }
+          l.copy(links = updatedLinks)
         case None =>
           val key = createKey(v)
           val id = Id[IndexedLink[D]](key)
@@ -58,3 +81,12 @@ case class IndexedLinks[V, D <: Document[D]](createKey: V => String,
 
   def query(value: V): fs2.Stream[IO, D] = queryIds(value).evalMap(collection.apply)
 }
+
+sealed trait MaxLinks
+
+object MaxLinks {
+  case object NoMax extends MaxLinks
+  case class OverflowError(max: Int = 1000) extends MaxLinks
+  case class OverflowWarn(max: Int = 1000) extends MaxLinks
+  case class OverflowTrim(max: Int = 1000) extends MaxLinks
+}