Skip to content

Commit

Permalink
Feature/updater provide more info about update (#657)
Browse files Browse the repository at this point in the history 
* Provide last global update timestamp

* Provide skipped mangas in update status

* Extract update status logic into function

* Rename update "statusMap" to "mangaStatusMap"

* Provide info about categories in update status
  • Loading branch information
@schroda
schroda authored Aug 15, 2023
1 parent d9019b8 commit 5baf543
Show file tree
Hide file tree
Showing 9 changed files with 94 additions and 36 deletions.
16 changes: 16 additions & 0 deletions server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/CategoryDataLoader.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,22 @@ class CategoryDataLoader : KotlinDataLoader<Int, CategoryType> {
}
}

class CategoryForIdsDataLoader : KotlinDataLoader<List<Int>, CategoryNodeList> {
override val dataLoaderName = "CategoryForIdsDataLoader"
override fun getDataLoader(): DataLoader<List<Int>, CategoryNodeList> = DataLoaderFactory.newDataLoader { categoryIds ->
future {
transaction {
addLogger(Slf4jSqlDebugLogger)
val ids = categoryIds.flatten().distinct()
val categories = CategoryTable.select { CategoryTable.id inList ids }.map { CategoryType(it) }
categoryIds.map { categoryIds ->
categories.filter { it.id in categoryIds }.toNodeList()
}
}
}
}
}

class CategoriesForMangaDataLoader : KotlinDataLoader<Int, CategoryNodeList> {
override val dataLoaderName = "CategoriesForMangaDataLoader"
override fun getDataLoader(): DataLoader<Int, CategoryNodeList> = DataLoaderFactory.newDataLoader<Int, CategoryNodeList> { ids ->
Expand Down
17 changes: 7 additions & 10 deletions server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,18 @@ import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import suwayomi.tachidesk.graphql.types.UpdateStatus
import suwayomi.tachidesk.graphql.types.UpdateStatusType
import suwayomi.tachidesk.manga.impl.update.IUpdater
import suwayomi.tachidesk.manga.impl.update.JobStatus

class UpdateQuery {
private val updater by DI.global.instance<IUpdater>()

fun updateStatus(): UpdateStatus {
val status = updater.status.value
return UpdateStatus(
isRunning = status.running,
pendingJobs = UpdateStatusType(status.statusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
runningJobs = UpdateStatusType(status.statusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
completeJobs = UpdateStatusType(status.statusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
failedJobs = UpdateStatusType(status.statusMap[JobStatus.FAILED]?.map { it.id }.orEmpty())
)
return UpdateStatus(updater.status.value)
}

data class LastUpdateTimestampPayload(val timestamp: Long)

fun lastUpdateTimestamp(): LastUpdateTimestampPayload {
return LastUpdateTimestampPayload(updater.getLastUpdateTimestamp())
}
}
2 changes: 2 additions & 0 deletions ...r/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ package suwayomi.tachidesk.graphql.server
import com.expediagroup.graphql.dataloader.KotlinDataLoaderRegistryFactory
import suwayomi.tachidesk.graphql.dataLoaders.CategoriesForMangaDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.CategoryDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.CategoryForIdsDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.CategoryMetaDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.ChapterDataLoader
import suwayomi.tachidesk.graphql.dataLoaders.ChapterMetaDataLoader
Expand Down Expand Up @@ -39,6 +40,7 @@ class TachideskDataLoaderRegistryFactory {
MangaForSourceDataLoader(),
MangaForIdsDataLoader(),
CategoryDataLoader(),
CategoryForIdsDataLoader(),
CategoryMetaDataLoader(),
CategoriesForMangaDataLoader(),
SourceDataLoader(),
Expand Down
26 changes: 21 additions & 5 deletions server/src/main/kotlin/suwayomi/tachidesk/graphql/types/UpdateType.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,26 +3,42 @@ package suwayomi.tachidesk.graphql.types
import com.expediagroup.graphql.generator.annotations.GraphQLIgnore
import com.expediagroup.graphql.server.extensions.getValueFromDataLoader
import graphql.schema.DataFetchingEnvironment
import suwayomi.tachidesk.manga.impl.update.CategoryUpdateStatus
import suwayomi.tachidesk.manga.impl.update.JobStatus
import suwayomi.tachidesk.manga.impl.update.UpdateStatus
import java.util.concurrent.CompletableFuture

class UpdateStatus(
val isRunning: Boolean,
val skippedCategories: UpdateStatusCategoryType,
val updatingCategories: UpdateStatusCategoryType,
val pendingJobs: UpdateStatusType,
val runningJobs: UpdateStatusType,
val completeJobs: UpdateStatusType,
val failedJobs: UpdateStatusType
val failedJobs: UpdateStatusType,
val skippedJobs: UpdateStatusType
) {
constructor(status: UpdateStatus) : this(
isRunning = status.running,
pendingJobs = UpdateStatusType(status.statusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
runningJobs = UpdateStatusType(status.statusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
completeJobs = UpdateStatusType(status.statusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
failedJobs = UpdateStatusType(status.statusMap[JobStatus.FAILED]?.map { it.id }.orEmpty())
skippedCategories = UpdateStatusCategoryType(status.categoryStatusMap[CategoryUpdateStatus.SKIPPED]?.map { it.id }.orEmpty()),
updatingCategories = UpdateStatusCategoryType(status.categoryStatusMap[CategoryUpdateStatus.UPDATING]?.map { it.id }.orEmpty()),
pendingJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
runningJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
completeJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
failedJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.FAILED]?.map { it.id }.orEmpty()),
skippedJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.SKIPPED]?.map { it.id }.orEmpty())
)
}

class UpdateStatusCategoryType(
@get:GraphQLIgnore
val categoryIds: List<Int>
) {
fun categories(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture<CategoryNodeList> {
return dataFetchingEnvironment.getValueFromDataLoader("CategoryForIdsDataLoader", categoryIds)
}
}

class UpdateStatusType(
@get:GraphQLIgnore
val mangaIds: List<Int>
Expand Down
1 change: 1 addition & 0 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.StateFlow
import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass

interface IUpdater {
fun getLastUpdateTimestamp(): Long
fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean?, forceAll: Boolean)
val status: StateFlow<UpdateStatus>
fun reset()
Expand Down
3 changes: 2 additions & 1 deletion server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ enum class JobStatus {
PENDING,
RUNNING,
COMPLETE,
FAILED
FAILED,
SKIPPED
}

data class UpdateJob(
Expand Down
17 changes: 11 additions & 6 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,27 @@
package suwayomi.tachidesk.manga.impl.update

import com.fasterxml.jackson.annotation.JsonIgnore
import mu.KotlinLogging
import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass

val logger = KotlinLogging.logger {}
enum class CategoryUpdateStatus {
UPDATING, SKIPPED
}

data class UpdateStatus(
val statusMap: Map<JobStatus, List<MangaDataClass>> = emptyMap(),
val categoryStatusMap: Map<CategoryUpdateStatus, List<CategoryDataClass>> = emptyMap(),
val mangaStatusMap: Map<JobStatus, List<MangaDataClass>> = emptyMap(),
val running: Boolean = false,
@JsonIgnore
val numberOfJobs: Int = 0
) {

constructor(jobs: List<UpdateJob>, running: Boolean) : this(
statusMap = jobs.groupBy { it.status }
constructor(categories: Map<CategoryUpdateStatus, List<CategoryDataClass>>, jobs: List<UpdateJob>, skippedMangas: List<MangaDataClass>, running: Boolean) : this(
categories,
mangaStatusMap = jobs.groupBy { it.status }
.mapValues { entry ->
entry.value.map { it.manga }
},
}.plus(Pair(JobStatus.SKIPPED, skippedMangas)),
running = running,
numberOfJobs = jobs.size
)
Expand Down
45 changes: 32 additions & 13 deletions server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,6 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import mu.KotlinLogging
import org.kodein.di.DI
import org.kodein.di.conf.global
import org.kodein.di.instance
import suwayomi.tachidesk.manga.impl.Category
import suwayomi.tachidesk.manga.impl.CategoryManga
import suwayomi.tachidesk.manga.impl.Chapter
Expand Down Expand Up @@ -49,6 +46,7 @@ class Updater : IUpdater {
private var maxSourcesInParallel = 20 // max permits, necessary to be set to be able to release up to 20 permits
private val semaphore = Semaphore(maxSourcesInParallel)

private val lastUpdateKey = "lastUpdateKey"
private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey"
private val preferences = Preferences.userNodeForPackage(Updater::class.java)

Expand Down Expand Up @@ -76,6 +74,10 @@ class Updater : IUpdater {
)
}

override fun getLastUpdateTimestamp(): Long {
return preferences.getLong(lastUpdateKey, 0)
}

private fun autoUpdateTask() {
val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
Expand Down Expand Up @@ -109,6 +111,15 @@ class Updater : IUpdater {
HAScheduler.schedule(::autoUpdateTask, updateInterval, timeToNextExecution, "global-update")
}

/**
* Updates the status and sustains the "skippedMangas"
*/
private fun updateStatus(jobs: List<UpdateJob>, running: Boolean, categories: Map<CategoryUpdateStatus, List<CategoryDataClass>>? = null, skippedMangas: List<MangaDataClass>? = null) {
val updateStatusCategories = categories ?: _status.value.categoryStatusMap
val tmpSkippedMangas = skippedMangas ?: _status.value.mangaStatusMap[JobStatus.SKIPPED] ?: emptyList()
_status.update { UpdateStatus(updateStatusCategories, jobs, tmpSkippedMangas, running) }
}

private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {
return updateChannels.getOrPut(source) {
logger.debug { "getOrCreateUpdateChannelFor: created channel for $source - channels: ${updateChannels.size + 1}" }
Expand All @@ -121,7 +132,7 @@ class Updater : IUpdater {
channel.consumeAsFlow()
.onEach { job ->
semaphore.withPermit {
_status.value = UpdateStatus(
updateStatus(
process(job),
tracker.any { (_, job) ->
job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING
Expand All @@ -136,7 +147,7 @@ class Updater : IUpdater {

private suspend fun process(job: UpdateJob): List<UpdateJob> {
tracker[job.manga.id] = job.copy(status = JobStatus.RUNNING)
_status.update { UpdateStatus(tracker.values.toList(), true) }
updateStatus(tracker.values.toList(), true)
tracker[job.manga.id] = try {
logger.info { "Updating \"${job.manga.title}\" (source: ${job.manga.sourceId})" }
Chapter.getChapterList(job.manga.id, true)
Expand All @@ -150,9 +161,10 @@ class Updater : IUpdater {
}

override fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean?, forceAll: Boolean) {
val updater by DI.global.instance<IUpdater>()
preferences.putLong(lastUpdateKey, System.currentTimeMillis())

if (clear == true) {
updater.reset()
reset()
}

val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
Expand All @@ -164,6 +176,11 @@ class Updater : IUpdater {
} else {
includedCategories.ifEmpty { unsetCategories }
}
val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
val updateStatusCategories = mapOf(
Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
Pair(CategoryUpdateStatus.SKIPPED, skippedCategories)
)

logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }

Expand All @@ -179,10 +196,12 @@ class Updater : IUpdater {
.filter { if (serverConfig.excludeCompleted.value) { it.status != MangaStatus.COMPLETED.name } else true }
.filter { forceAll || !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
.toList()
val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()

// In case no manga gets updated and no update job was running before, the client would never receive an info about its update request
updateStatus(emptyList(), mangasToUpdate.isNotEmpty(), updateStatusCategories, skippedMangas)

if (mangasToUpdate.isEmpty()) {
UpdaterSocket.notifyAllClients(UpdateStatus())
return
}

Expand All @@ -192,10 +211,10 @@ class Updater : IUpdater {
)
}

private fun addMangasToQueue(mangas: List<MangaDataClass>) {
mangas.forEach { tracker[it.id] = UpdateJob(it) }
_status.update { UpdateStatus(tracker.values.toList(), mangas.isNotEmpty()) }
mangas.forEach { addMangaToQueue(it) }
private fun addMangasToQueue(mangasToUpdate: List<MangaDataClass>) {
mangasToUpdate.forEach { tracker[it.id] = UpdateJob(it) }
updateStatus(tracker.values.toList(), mangasToUpdate.isNotEmpty())
mangasToUpdate.forEach { addMangaToQueue(it) }
}

private fun addMangaToQueue(manga: MangaDataClass) {
Expand All @@ -208,7 +227,7 @@ class Updater : IUpdater {
override fun reset() {
scope.coroutineContext.cancelChildren()
tracker.clear()
_status.update { UpdateStatus() }
updateStatus(emptyList(), false)
updateChannels.forEach { (_, channel) -> channel.cancel() }
updateChannels.clear()
}
Expand Down
3 changes: 2 additions & 1 deletion server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,8 @@ enum class WebUIChannel {
}

enum class WebUIFlavor(
val uiName: String, val repoUrl: String,
val uiName: String,
val repoUrl: String,
val versionMappingUrl: String,
val latestReleaseInfoUrl: String,
val baseFileName: String
Expand Down

0 comments on commit 5baf543

Please sign in to comment.