From 5e66b57fccb4f1fba16a179e6ede8c7da145ca4b Mon Sep 17 00:00:00 2001 From: Syer10 Date: Fri, 3 Nov 2023 19:53:49 -0400 Subject: [PATCH 1/2] Use new extension icon path --- .../tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt index b19ce9aa5..e0ac99f8b 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/github/ExtensionGithubApi.kt @@ -109,7 +109,7 @@ object ExtensionGithubApi { hasChangelog = it.hasChangelog == 1, sources = it.sources?.toExtensionSources() ?: emptyList(), apkName = it.apk, - iconUrl = "${REPO_URL_PREFIX}icon/${it.apk.replace(".apk", ".png")}", + iconUrl = "${REPO_URL_PREFIX}icon/${it.pkg}.png", ) } } From b7940481807610599d56a67db7ec973049e19bb4 Mon Sep 17 00:00:00 2001 From: Syer10 Date: Fri, 3 Nov 2023 19:54:10 -0400 Subject: [PATCH 2/2] Improve Extension list performance --- .../manga/impl/extension/ExtensionsList.kt | 140 +++++++++++------- 1 file changed, 87 insertions(+), 53 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt index 728871cc3..d392075f8 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/extension/ExtensionsList.kt @@ -9,11 +9,13 @@ package suwayomi.tachidesk.manga.impl.extension import eu.kanade.tachiyomi.source.local.LocalSource import mu.KotlinLogging -import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq +import org.jetbrains.exposed.dao.id.EntityID +import org.jetbrains.exposed.sql.ResultRow +import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList +import org.jetbrains.exposed.sql.batchInsert import org.jetbrains.exposed.sql.deleteWhere -import org.jetbrains.exposed.sql.insert -import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.selectAll +import org.jetbrains.exposed.sql.statements.BatchUpdateStatement import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.manga.impl.extension.Extension.getExtensionIconUrl @@ -69,68 +71,100 @@ object ExtensionsList { private fun updateExtensionDatabase(foundExtensions: List) { transaction { - foundExtensions.forEach { foundExtension -> - val extensionRecord = ExtensionTable.select { ExtensionTable.pkgName eq foundExtension.pkgName }.firstOrNull() - if (extensionRecord != null) { - if (extensionRecord[ExtensionTable.isInstalled]) { - when { - foundExtension.versionCode > extensionRecord[ExtensionTable.versionCode] -> { - // there is an update - ExtensionTable.update({ ExtensionTable.pkgName eq foundExtension.pkgName }) { - it[hasUpdate] = true + val installedExtensions = + ExtensionTable.selectAll().toList() + .associateBy { it[ExtensionTable.pkgName] } + val extensionsToUpdate = mutableListOf>() + val extensionsToInsert = mutableListOf() + val extensionsToDelete = + installedExtensions.mapNotNull { (pkgName, extension) -> + extension.takeUnless { foundExtensions.any { it.pkgName == pkgName } } + } + foundExtensions.forEach { + val extension = installedExtensions[it.pkgName] + if (extension != null) { + extensionsToUpdate.add(it to extension) + } else { + extensionsToInsert.add(it) + } + } + if (extensionsToUpdate.isNotEmpty()) { + val extensionsInstalled = + extensionsToUpdate + .groupBy { it.second[ExtensionTable.isInstalled] } + val installedExtensionsToUpdate = extensionsInstalled[true].orEmpty() + if (installedExtensionsToUpdate.isNotEmpty()) { + BatchUpdateStatement(ExtensionTable).apply { + installedExtensionsToUpdate.forEach { (foundExtension, extensionRecord) -> + addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) + // Always update icon url + this[ExtensionTable.iconUrl] = foundExtension.iconUrl + + // add these because batch updates need matching columns + this[ExtensionTable.hasUpdate] = extensionRecord[ExtensionTable.hasUpdate] + this[ExtensionTable.isObsolete] = extensionRecord[ExtensionTable.isObsolete] + when { + foundExtension.versionCode > extensionRecord[ExtensionTable.versionCode] -> { + // there is an update + this[ExtensionTable.hasUpdate] = true + updateMap.putIfAbsent(foundExtension.pkgName, foundExtension) } - updateMap.putIfAbsent(foundExtension.pkgName, foundExtension) - } - foundExtension.versionCode < extensionRecord[ExtensionTable.versionCode] -> { - // somehow the user installed an invalid version - ExtensionTable.update({ ExtensionTable.pkgName eq foundExtension.pkgName }) { - it[isObsolete] = true + foundExtension.versionCode < extensionRecord[ExtensionTable.versionCode] -> { + // somehow the user installed an invalid version + this[ExtensionTable.isObsolete] = true } } } - } else { - // extension is not installed, so we can overwrite the data without a care - ExtensionTable.update({ ExtensionTable.pkgName eq foundExtension.pkgName }) { - it[name] = foundExtension.name - it[versionName] = foundExtension.versionName - it[versionCode] = foundExtension.versionCode - it[lang] = foundExtension.lang - it[isNsfw] = foundExtension.isNsfw - it[apkName] = foundExtension.apkName - it[iconUrl] = foundExtension.iconUrl - } + execute(this@transaction) } - } else { - // insert new record - ExtensionTable.insert { - it[name] = foundExtension.name - it[pkgName] = foundExtension.pkgName - it[versionName] = foundExtension.versionName - it[versionCode] = foundExtension.versionCode - it[lang] = foundExtension.lang - it[isNsfw] = foundExtension.isNsfw - it[apkName] = foundExtension.apkName - it[iconUrl] = foundExtension.iconUrl + } + val extensionsToFullyUpdate = extensionsInstalled[false].orEmpty() + if (extensionsToFullyUpdate.isNotEmpty()) { + BatchUpdateStatement(ExtensionTable).apply { + extensionsToFullyUpdate.forEach { (foundExtension, extensionRecord) -> + addBatch(EntityID(extensionRecord[ExtensionTable.id].value, ExtensionTable)) + // extension is not installed, so we can overwrite the data without a care + this[ExtensionTable.name] = foundExtension.name + this[ExtensionTable.versionName] = foundExtension.versionName + this[ExtensionTable.versionCode] = foundExtension.versionCode + this[ExtensionTable.lang] = foundExtension.lang + this[ExtensionTable.isNsfw] = foundExtension.isNsfw + this[ExtensionTable.apkName] = foundExtension.apkName + this[ExtensionTable.iconUrl] = foundExtension.iconUrl + } + execute(this@transaction) } } } + if (extensionsToInsert.isNotEmpty()) { + ExtensionTable.batchInsert(extensionsToInsert) { foundExtension -> + this[ExtensionTable.name] = foundExtension.name + this[ExtensionTable.pkgName] = foundExtension.pkgName + this[ExtensionTable.versionName] = foundExtension.versionName + this[ExtensionTable.versionCode] = foundExtension.versionCode + this[ExtensionTable.lang] = foundExtension.lang + this[ExtensionTable.isNsfw] = foundExtension.isNsfw + this[ExtensionTable.apkName] = foundExtension.apkName + this[ExtensionTable.iconUrl] = foundExtension.iconUrl + } + } // deal with obsolete extensions - ExtensionTable.selectAll().forEach { extensionRecord -> - val foundExtension = foundExtensions.find { it.pkgName == extensionRecord[ExtensionTable.pkgName] } - if (foundExtension == null) { - // not in the repo, so these extensions are obsolete - if (extensionRecord[ExtensionTable.isInstalled]) { - // is installed so we should mark it as obsolete - ExtensionTable.update({ ExtensionTable.pkgName eq extensionRecord[ExtensionTable.pkgName] }) { - it[isObsolete] = true - } - } else { - // is not installed, so we can remove the record without a care - ExtensionTable.deleteWhere { ExtensionTable.pkgName eq extensionRecord[ExtensionTable.pkgName] } - } + val extensionsToRemove = + extensionsToDelete.groupBy { it[ExtensionTable.isInstalled] } + .mapValues { (_, extensions) -> extensions.map { it[ExtensionTable.pkgName] } } + // not in the repo, so these extensions are obsolete + val obsoleteExtensions = extensionsToRemove[true].orEmpty() + if (obsoleteExtensions.isNotEmpty()) { + ExtensionTable.update({ ExtensionTable.pkgName inList obsoleteExtensions }) { + it[isObsolete] = true } } + // is not installed, so we can remove the record without a care + val removeExtensions = extensionsToRemove[false].orEmpty() + if (removeExtensions.isNotEmpty()) { + ExtensionTable.deleteWhere { ExtensionTable.pkgName inList removeExtensions } + } } } }