Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/improve browsing source performance #743

Merged
merged 2 commits into from
Oct 30, 2023
Merged

Feature/improve browsing source performance #743

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
532ad6a
Improve browse source database operations
schroda Oct 29, 2023
1627e68
Reuse "insertOrGet" for "processEntries"
schroda Oct 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 32 additions & 109 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/MangaList.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,14 @@ package suwayomi.tachidesk.manga.impl
* file, You can obtain one at https://mozilla.org/MPL/2.0/. */

import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.UpdateStrategy
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.insertAndGetId
import org.jetbrains.exposed.sql.batchInsert
import org.jetbrains.exposed.sql.select
import org.jetbrains.exposed.sql.transactions.transaction
import suwayomi.tachidesk.manga.impl.Manga.getMangaMetaMap
import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
import suwayomi.tachidesk.manga.model.dataclass.PagedMangaListDataClass
import suwayomi.tachidesk.manga.model.dataclass.toGenreList
import suwayomi.tachidesk.manga.model.table.MangaStatus
import suwayomi.tachidesk.manga.model.table.MangaTable
import suwayomi.tachidesk.manga.model.table.toDataClass

object MangaList {
fun proxyThumbnailUrl(mangaId: Int): String {
Expand Down Expand Up @@ -50,35 +46,38 @@ object MangaList {

fun MangasPage.insertOrGet(sourceId: Long): List<Int> {
return transaction {
mangas.map { manga ->
val mangaEntry =
MangaTable.select {
(MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
}.firstOrNull()
if (mangaEntry == null) { // create manga entry
val mangaId =
MangaTable.insertAndGetId {
it[url] = manga.url
it[title] = manga.title
val existingMangaUrlsToId =
MangaTable.slice(MangaTable.url, MangaTable.id).select {
(MangaTable.sourceReference eq sourceId) and (MangaTable.url inList mangas.map { it.url })
}.associate { Pair(it[MangaTable.url], it[MangaTable.id].value) }
val existingMangaUrls = existingMangaUrlsToId.map { it.key }

it[artist] = manga.artist
it[author] = manga.author
it[description] = manga.description
it[genre] = manga.genre
it[status] = manga.status
it[thumbnail_url] = manga.thumbnail_url
it[updateStrategy] = manga.update_strategy.name
val mangasToInsert = mangas.filter { !existingMangaUrls.contains(it.url) }

it[sourceReference] = sourceId
}.value
val insertedMangaUrlsToId =
MangaTable.batchInsert(mangasToInsert) {
this[MangaTable.url] = it.url
this[MangaTable.title] = it.title

// delete thumbnail in case cached data still exists
Manga.clearThumbnail(mangaId)
this[MangaTable.artist] = it.artist
this[MangaTable.author] = it.author
this[MangaTable.description] = it.description
this[MangaTable.genre] = it.genre
this[MangaTable.status] = it.status
this[MangaTable.thumbnail_url] = it.thumbnail_url
this[MangaTable.updateStrategy] = it.update_strategy.name

mangaId
} else {
mangaEntry[MangaTable.id].value
}
this[MangaTable.sourceReference] = sourceId
}.associate { Pair(it[MangaTable.url], it[MangaTable.id].value) }

// delete thumbnail in case cached data still exists
insertedMangaUrlsToId.forEach { (_, id) -> Manga.clearThumbnail(id) }

val mangaUrlsToId = existingMangaUrlsToId + insertedMangaUrlsToId

mangas.map { manga ->
mangaUrlsToId[manga.url]
?: throw Exception("MangaList::insertOrGet($sourceId): Something went wrong inserting browsed source mangas")
}
}
}
Expand All @@ -87,84 +86,8 @@ object MangaList {
val mangasPage = this
val mangaList =
transaction {
return@transaction mangasPage.mangas.map { manga ->
var mangaEntry =
MangaTable.select {
(MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
}.firstOrNull()
if (mangaEntry == null) { // create manga entry
val mangaId =
MangaTable.insertAndGetId {
it[url] = manga.url
it[title] = manga.title

it[artist] = manga.artist
it[author] = manga.author
it[description] = manga.description
it[genre] = manga.genre
it[status] = manga.status
it[thumbnail_url] = manga.thumbnail_url
it[updateStrategy] = manga.update_strategy.name

it[sourceReference] = sourceId
}.value

// delete thumbnail in case cached data still exists
Manga.clearThumbnail(mangaId)

mangaEntry =
MangaTable.select {
(MangaTable.url eq manga.url) and (MangaTable.sourceReference eq sourceId)
}.first()

MangaDataClass(
id = mangaId,
sourceId = sourceId.toString(),
url = manga.url,
title = manga.title,
thumbnailUrl = proxyThumbnailUrl(mangaId),
thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
initialized = manga.initialized,
artist = manga.artist,
author = manga.author,
description = manga.description,
genre = manga.genre.toGenreList(),
status = MangaStatus.valueOf(manga.status).name,
inLibrary = false, // It's a new manga entry
inLibraryAt = 0,
meta = getMangaMetaMap(mangaId),
realUrl = mangaEntry[MangaTable.realUrl],
lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
freshData = true,
)
} else {
val mangaId = mangaEntry[MangaTable.id].value
MangaDataClass(
id = mangaId,
sourceId = sourceId.toString(),
url = manga.url,
title = manga.title,
thumbnailUrl = proxyThumbnailUrl(mangaId),
thumbnailUrlLastFetched = mangaEntry[MangaTable.thumbnailUrlLastFetched],
initialized = true,
artist = mangaEntry[MangaTable.artist],
author = mangaEntry[MangaTable.author],
description = mangaEntry[MangaTable.description],
genre = mangaEntry[MangaTable.genre].toGenreList(),
status = MangaStatus.valueOf(mangaEntry[MangaTable.status]).name,
inLibrary = mangaEntry[MangaTable.inLibrary],
inLibraryAt = mangaEntry[MangaTable.inLibraryAt],
meta = getMangaMetaMap(mangaId),
realUrl = mangaEntry[MangaTable.realUrl],
lastFetchedAt = mangaEntry[MangaTable.lastFetchedAt],
chaptersLastFetchedAt = mangaEntry[MangaTable.chaptersLastFetchedAt],
updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
freshData = false,
)
}
}
val mangaIds = insertOrGet(sourceId)
return@transaction MangaTable.select { MangaTable.id inList mangaIds }.map { MangaTable.toDataClass(it) }
}
return PagedMangaListDataClass(
mangaList,
Expand Down
Loading