From 4490bfb05a96d1468636add95dd91a369e2bade4 Mon Sep 17 00:00:00 2001 From: akabhirav Date: Thu, 9 Feb 2023 16:36:15 +0530 Subject: [PATCH 1/6] Separate cache dir from download dir --- .../suwayomi/tachidesk/manga/impl/Page.kt | 2 +- .../tachidesk/manga/impl/util/DirName.kt | 18 +++++++++++------- .../manga/impl/util/storage/ImageResponse.kt | 16 +++++++++++++--- .../suwayomi/tachidesk/server/ServerSetup.kt | 1 + 4 files changed, 26 insertions(+), 11 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt index 96c4d32db..41434f6b7 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt @@ -86,7 +86,7 @@ object Page { File(chapterDir).mkdirs() val fileName = getPageName(index) - return getImageResponse(chapterDir, fileName, useCache) { + return getImageResponse(mangaId, chapterId, fileName, useCache) { source.fetchImage(tachiyomiPage).awaitSingle() } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt index f19f0f63f..4c9b0fd75 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt @@ -7,6 +7,7 @@ package suwayomi.tachidesk.manga.impl.util * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ +import org.jetbrains.exposed.sql.ResultRow import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import org.kodein.di.DI @@ -21,17 +22,16 @@ import java.io.File private val applicationDirs by DI.global.instance() -fun getMangaDir(mangaId: Int): String { - val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() } +fun getMangaDir(mangaId: Int, cache: Boolean = false): String { + val mangaEntry = getMangaEntry(mangaId) val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) val sourceDir = source.toString() val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title]) - - return "${applicationDirs.mangaDownloadsRoot}/$sourceDir/$mangaDir" + return (if (cache) applicationDirs.cacheRoot else applicationDirs.mangaDownloadsRoot) + "/$sourceDir/$mangaDir" } -fun getChapterDir(mangaId: Int, chapterId: Int): String { +fun getChapterDir(mangaId: Int, chapterId: Int, cache: Boolean = false): String { val chapterEntry = transaction { ChapterTable.select { ChapterTable.id eq chapterId }.first() } val chapterDir = SafePath.buildValidFilename( @@ -41,12 +41,12 @@ fun getChapterDir(mangaId: Int, chapterId: Int): String { } ) - return getMangaDir(mangaId) + "/$chapterDir" + return getMangaDir(mangaId, cache) + "/$chapterDir" } /** return value says if rename/move was successful */ fun updateMangaDownloadDir(mangaId: Int, newTitle: String): Boolean { - val mangaEntry = transaction { MangaTable.select { MangaTable.id eq mangaId }.first() } + val mangaEntry = getMangaEntry(mangaId) val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference]) val sourceDir = source.toString() @@ -66,3 +66,7 @@ fun updateMangaDownloadDir(mangaId: Int, newTitle: String): Boolean { true } } + +private fun getMangaEntry(mangaId: Int): ResultRow { + return transaction { MangaTable.select { MangaTable.id eq mangaId }.first() } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt index 2c8d9cc14..5bd02f152 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt @@ -9,6 +9,7 @@ package suwayomi.tachidesk.manga.impl.util.storage import okhttp3.Response import okhttp3.internal.closeQuietly +import suwayomi.tachidesk.manga.impl.util.getChapterDir import java.io.File import java.io.FileInputStream import java.io.InputStream @@ -29,7 +30,7 @@ object ImageResponse { } /** fetch a cached image response, calls `fetcher` if cache fails */ - suspend fun getCachedImageResponse(saveDir: String, fileName: String, fetcher: suspend () -> Response): Pair { + private suspend fun getCachedImageResponse(saveDir: String, fileName: String, fetcher: suspend () -> Response): Pair { val cachedFile = findFileNameStartingWith(saveDir, fileName) val filePath = "$saveDir/$fileName" if (cachedFile != null) { @@ -70,7 +71,7 @@ object ImageResponse { } } - suspend fun getNoCacheImageResponse(fetcher: suspend () -> Response): Pair { + private suspend fun getNoCacheImageResponse(fetcher: suspend () -> Response): Pair { val response = fetcher() if (response.code == 200) { @@ -88,11 +89,20 @@ object ImageResponse { } } - suspend fun getImageResponse(saveDir: String, fileName: String, useCache: Boolean = false, fetcher: suspend () -> Response): Pair { + suspend fun getImageResponse(saveDir: String, fileName: String, useCache: Boolean, fetcher: suspend () -> Response): Pair { return if (useCache) { getCachedImageResponse(saveDir, fileName, fetcher) } else { getNoCacheImageResponse(fetcher) } } + + suspend fun getImageResponse(mangaId: Int, chapterId: Int, fileName: String, useCache: Boolean, fetcher: suspend () -> Response): Pair { + var saveDir = "" + if (useCache) { + saveDir = getChapterDir(mangaId, chapterId, true) + File(saveDir).mkdir() + } + return getImageResponse(saveDir, fileName, useCache, fetcher) + } } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt index e5c04e03a..c7fc1f4b9 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt @@ -38,6 +38,7 @@ private val logger = KotlinLogging.logger {} class ApplicationDirs( val dataRoot: String = ApplicationRootDir ) { + val cacheRoot = System.getProperty("java.io.tmpdir") + "/tachidesk" val extensionsRoot = "$dataRoot/extensions" val thumbnailsRoot = "$dataRoot/thumbnails" val mangaDownloadsRoot = serverConfig.downloadsPath.ifBlank { "$dataRoot/downloads" } From df9c8c1f984ec6e881c841e6f49c9e27c9e3cbff Mon Sep 17 00:00:00 2001 From: akabhirav Date: Thu, 9 Feb 2023 19:46:28 +0530 Subject: [PATCH 2/6] Move downloader logic outside of caching/image download logic --- .../suwayomi/tachidesk/manga/impl/Chapter.kt | 5 +-- .../manga/impl/ChapterDownloadHelper.kt | 24 ++++++++++++ .../suwayomi/tachidesk/manga/impl/Page.kt | 7 ++-- .../impl/download/DownloadedFilesProvider.kt | 18 +++++++++ .../manga/impl/download/Downloader.kt | 7 +++- .../manga/impl/download/FolderProvider.kt | 37 +++++++++++++++++++ .../manga/impl/util/storage/ImageResponse.kt | 31 +++++++++------- 7 files changed, 106 insertions(+), 23 deletions(-) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index fa8739293..02f41b52d 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -15,7 +15,6 @@ import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.sql.* import org.jetbrains.exposed.sql.SortOrder.ASC import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq -import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList import org.jetbrains.exposed.sql.transactions.transaction import suwayomi.tachidesk.manga.impl.Manga.getManga import suwayomi.tachidesk.manga.impl.util.getChapterDir @@ -312,9 +311,7 @@ object Chapter { ChapterTable.select { (ChapterTable.manga eq mangaId) and (ChapterTable.sourceOrder eq chapterIndex) } .first()[ChapterTable.id].value - val chapterDir = getChapterDir(mangaId, chapterId) - - File(chapterDir).deleteRecursively() + ChapterDownloadHelper.delete(mangaId, chapterId) ChapterTable.update({ (ChapterTable.manga eq mangaId) and (ChapterTable.sourceOrder eq chapterIndex) }) { it[isDownloaded] = false diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt new file mode 100644 index 000000000..b97fb0f9c --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt @@ -0,0 +1,24 @@ +package suwayomi.tachidesk.manga.impl + +import suwayomi.tachidesk.manga.impl.download.DownloadedFilesProvider +import suwayomi.tachidesk.manga.impl.download.FolderProvider +import java.io.InputStream + +object ChapterDownloadHelper { + fun getImage(mangaId: Int, chapterId: Int, index: Int): Pair { + return provider(mangaId, chapterId).getImage(index) + } + + fun delete(mangaId: Int, chapterId: Int): Boolean { + return provider(mangaId, chapterId).delete() + } + + fun putImage(mangaId: Int, chapterId: Int, index: Int, image: InputStream): Boolean { + return provider(mangaId, chapterId).putImage(index, image) + } + + // return the appropriate provider based on how the download was saved. For the logic is simple but will evolve when new types of downloads are available + private fun provider(mangaId: Int, chapterId: Int): DownloadedFilesProvider { + return FolderProvider(mangaId, chapterId) + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt index 41434f6b7..4a146b245 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt @@ -15,7 +15,6 @@ import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.select import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update -import suwayomi.tachidesk.manga.impl.util.getChapterDir import suwayomi.tachidesk.manga.impl.util.lang.awaitSingle import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource.getCatalogueSourceOrStub import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse.getImageResponse @@ -82,10 +81,12 @@ object Page { } } - val chapterDir = getChapterDir(mangaId, chapterId) - File(chapterDir).mkdirs() val fileName = getPageName(index) + if (chapterEntry[ChapterTable.isDownloaded]) { + return ChapterDownloadHelper.getImage(mangaId, chapterId, index) + } + return getImageResponse(mangaId, chapterId, fileName, useCache) { source.fetchImage(tachiyomiPage).awaitSingle() } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt new file mode 100644 index 000000000..5be1a9844 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt @@ -0,0 +1,18 @@ +package suwayomi.tachidesk.manga.impl.download + +import java.io.InputStream + +/* +* Base class for downloaded chapter files provider, example: Folder, Archive +* */ +abstract class DownloadedFilesProvider(val mangaId: Int, val chapterId: Int) { + abstract fun getImage(index: Int): Pair + + abstract fun putImage(index: Int, image: InputStream): Boolean + + abstract fun delete(): Boolean + + private fun getPageName(index: Int): String { + return String.format("%03d", index + 1) + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt index c1cb19833..a2d66c5cc 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt @@ -23,6 +23,7 @@ import mu.KotlinLogging import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update +import suwayomi.tachidesk.manga.impl.ChapterDownloadHelper import suwayomi.tachidesk.manga.impl.Page.getPageImage import suwayomi.tachidesk.manga.impl.chapter.getChapterDownloadReady import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter @@ -99,7 +100,7 @@ class Downloader( for (pageNum in 0 until pageCount) { var pageProgressJob: Job? = null try { - getPageImage( + val image = getPageImage( mangaId = download.mangaId, chapterIndex = download.chapterIndex, index = pageNum, @@ -113,7 +114,9 @@ class Downloader( } .launchIn(scope) } - ).first.close() + ).first + ChapterDownloadHelper.putImage(download.mangaId, download.chapter.id, pageNum, image) + image.close() } finally { // always cancel the page progress job even if it throws an exception to avoid memory leaks pageProgressJob?.cancel() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt new file mode 100644 index 000000000..6e220af28 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt @@ -0,0 +1,37 @@ +package suwayomi.tachidesk.manga.impl.download + +import suwayomi.tachidesk.manga.impl.Page.getPageName +import suwayomi.tachidesk.manga.impl.util.getChapterDir +import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse +import java.io.File +import java.io.FileInputStream +import java.io.InputStream + +/* +* Provides downloaded files when pages were downloaded into folders +* */ +class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(mangaId, chapterId) { + override fun getImage(index: Int): Pair { + val chapterDir = getChapterDir(mangaId, chapterId) + val folder = File(chapterDir) + folder.mkdirs() + val file = folder.listFiles()?.get(index) + val fileType = file!!.name.substringAfterLast(".") + return Pair(FileInputStream(file).buffered(), "image/$fileType") + } + + override fun putImage(index: Int, image: InputStream): Boolean { + val chapterDir = getChapterDir(mangaId, chapterId) + val folder = File(chapterDir) + folder.mkdirs() + val fileName = getPageName(index) + val filePath = "$chapterDir/$fileName" + ImageResponse.saveImage(filePath, image) + return true + } + + override fun delete(): Boolean { + val chapterDir = getChapterDir(mangaId, chapterId) + return File(chapterDir).deleteRecursively() + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt index 5bd02f152..1499b5a04 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/storage/ImageResponse.kt @@ -44,19 +44,7 @@ object ImageResponse { val response = fetcher() if (response.code == 200) { - val tmpSavePath = "$filePath.tmp" - val tmpSaveFile = File(tmpSavePath) - response.body!!.source().saveTo(tmpSaveFile) - - // find image type - val imageType = response.headers["content-type"] - ?: ImageUtil.findImageType { tmpSaveFile.inputStream() }?.mime - ?: "image/jpeg" - - val actualSavePath = "$filePath.${imageType.substringAfter("/")}" - - tmpSaveFile.renameTo(File(actualSavePath)) - + val (actualSavePath, imageType) = saveImage(filePath, response.body!!.byteStream()) return pathToInputStream(actualSavePath) to imageType } else { response.closeQuietly() @@ -64,6 +52,21 @@ object ImageResponse { } } + fun saveImage(filePath: String, image: InputStream): Pair { + val tmpSavePath = "$filePath.tmp" + val tmpSaveFile = File(tmpSavePath) + image.use { input -> tmpSaveFile.outputStream().use { output -> input.copyTo(output) } } + + // find image type + val imageType = ImageUtil.findImageType { tmpSaveFile.inputStream() }?.mime + ?: "image/jpeg" + + val actualSavePath = "$filePath.${imageType.substringAfter("/")}" + + tmpSaveFile.renameTo(File(actualSavePath)) + return Pair(actualSavePath, imageType) + } + fun clearCachedImage(saveDir: String, fileName: String) { val cachedFile = findFileNameStartingWith(saveDir, fileName) cachedFile?.also { @@ -101,7 +104,7 @@ object ImageResponse { var saveDir = "" if (useCache) { saveDir = getChapterDir(mangaId, chapterId, true) - File(saveDir).mkdir() + File(saveDir).mkdirs() } return getImageResponse(saveDir, fileName, useCache, fetcher) } From 290eef076b450c75fd0052296e67aec37d4cdc96 Mon Sep 17 00:00:00 2001 From: akabhirav Date: Thu, 9 Feb 2023 20:35:29 +0530 Subject: [PATCH 3/6] remove unnecessary method duplication --- .../tachidesk/manga/impl/download/DownloadedFilesProvider.kt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt index 5be1a9844..ccfe5b383 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt @@ -11,8 +11,4 @@ abstract class DownloadedFilesProvider(val mangaId: Int, val chapterId: Int) { abstract fun putImage(index: Int, image: InputStream): Boolean abstract fun delete(): Boolean - - private fun getPageName(index: Int): String { - return String.format("%03d", index + 1) - } } From 8e891ba7da6c94e7b25bfa8d210f5102025d2804 Mon Sep 17 00:00:00 2001 From: akabhirav Date: Fri, 10 Feb 2023 12:22:11 +0530 Subject: [PATCH 4/6] moved download logic inside download provider --- .../manga/impl/ChapterDownloadHelper.kt | 13 +++- .../impl/download/DownloadedFilesProvider.kt | 9 ++- .../manga/impl/download/Downloader.kt | 35 +---------- .../manga/impl/download/FolderProvider.kt | 61 ++++++++++++++++--- 4 files changed, 74 insertions(+), 44 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt index b97fb0f9c..a80c37228 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt @@ -1,8 +1,11 @@ package suwayomi.tachidesk.manga.impl +import kotlinx.coroutines.CoroutineScope import suwayomi.tachidesk.manga.impl.download.DownloadedFilesProvider import suwayomi.tachidesk.manga.impl.download.FolderProvider +import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import java.io.InputStream +import kotlin.reflect.KSuspendFunction2 object ChapterDownloadHelper { fun getImage(mangaId: Int, chapterId: Int, index: Int): Pair { @@ -13,8 +16,14 @@ object ChapterDownloadHelper { return provider(mangaId, chapterId).delete() } - fun putImage(mangaId: Int, chapterId: Int, index: Int, image: InputStream): Boolean { - return provider(mangaId, chapterId).putImage(index, image) + suspend fun download( + mangaId: Int, + chapterId: Int, + download: DownloadChapter, + scope: CoroutineScope, + step: KSuspendFunction2 + ): Boolean { + return provider(mangaId, chapterId).download(download, scope, step) } // return the appropriate provider based on how the download was saved. For the logic is simple but will evolve when new types of downloads are available diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt index ccfe5b383..7f40f7ed4 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt @@ -1,6 +1,9 @@ package suwayomi.tachidesk.manga.impl.download +import kotlinx.coroutines.CoroutineScope +import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import java.io.InputStream +import kotlin.reflect.KSuspendFunction2 /* * Base class for downloaded chapter files provider, example: Folder, Archive @@ -8,7 +11,11 @@ import java.io.InputStream abstract class DownloadedFilesProvider(val mangaId: Int, val chapterId: Int) { abstract fun getImage(index: Int): Pair - abstract fun putImage(index: Int, image: InputStream): Boolean + abstract suspend fun download( + download: DownloadChapter, + scope: CoroutineScope, + step: KSuspendFunction2 + ): Boolean abstract fun delete(): Boolean } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt index a2d66c5cc..ddcc549a9 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/Downloader.kt @@ -13,10 +13,6 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.ensureActive -import kotlinx.coroutines.flow.distinctUntilChanged -import kotlinx.coroutines.flow.launchIn -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.sample import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import mu.KotlinLogging @@ -24,7 +20,6 @@ import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.manga.impl.ChapterDownloadHelper -import suwayomi.tachidesk.manga.impl.Page.getPageImage import suwayomi.tachidesk.manga.impl.chapter.getChapterDownloadReady import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import suwayomi.tachidesk.manga.impl.download.model.DownloadState.Downloading @@ -96,35 +91,7 @@ class Downloader( download.chapter = getChapterDownloadReady(download.chapterIndex, download.mangaId) step(download, false) - val pageCount = download.chapter.pageCount - for (pageNum in 0 until pageCount) { - var pageProgressJob: Job? = null - try { - val image = getPageImage( - mangaId = download.mangaId, - chapterIndex = download.chapterIndex, - index = pageNum, - progressFlow = { flow -> - pageProgressJob = flow - .sample(100) - .distinctUntilChanged() - .onEach { - download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount - step(null, false) // don't throw on canceled download here since we can't do anything - } - .launchIn(scope) - } - ).first - ChapterDownloadHelper.putImage(download.mangaId, download.chapter.id, pageNum, image) - image.close() - } finally { - // always cancel the page progress job even if it throws an exception to avoid memory leaks - pageProgressJob?.cancel() - } - // TODO: retry on error with 2,4,8 seconds of wait - download.progress = ((pageNum + 1).toFloat()) / pageCount - step(download, false) - } + ChapterDownloadHelper.download(download.mangaId, download.chapter.id, download, scope, this::step) download.state = Finished transaction { ChapterTable.update({ (ChapterTable.manga eq download.mangaId) and (ChapterTable.sourceOrder eq download.chapterIndex) }) { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt index 6e220af28..fac46f711 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt @@ -1,11 +1,23 @@ package suwayomi.tachidesk.manga.impl.download +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.launchIn +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.sample +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.withContext +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import suwayomi.tachidesk.manga.impl.Page import suwayomi.tachidesk.manga.impl.Page.getPageName +import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import suwayomi.tachidesk.manga.impl.util.getChapterDir import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse import java.io.File import java.io.FileInputStream import java.io.InputStream +import kotlin.reflect.KSuspendFunction2 /* * Provides downloaded files when pages were downloaded into folders @@ -20,13 +32,48 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man return Pair(FileInputStream(file).buffered(), "image/$fileType") } - override fun putImage(index: Int, image: InputStream): Boolean { - val chapterDir = getChapterDir(mangaId, chapterId) - val folder = File(chapterDir) - folder.mkdirs() - val fileName = getPageName(index) - val filePath = "$chapterDir/$fileName" - ImageResponse.saveImage(filePath, image) + @OptIn(FlowPreview::class) + override suspend fun download( + download: DownloadChapter, + scope: CoroutineScope, + step: KSuspendFunction2 + ): Boolean { + val pageCount = download.chapter.pageCount + for (pageNum in 0 until pageCount) { + var pageProgressJob: Job? = null + try { + val image = Page.getPageImage( + mangaId = download.mangaId, + chapterIndex = download.chapterIndex, + index = pageNum, + progressFlow = { flow -> + pageProgressJob = flow + .sample(100) + .distinctUntilChanged() + .onEach { + download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount + step(null, false) // don't throw on canceled download here since we can't do anything + } + .launchIn(scope) + } + ).first + val chapterDir = getChapterDir(mangaId, chapterId) + val folder = File(chapterDir) + folder.mkdirs() + val fileName = getPageName(pageNum) // might have to change this to index stored in database + val filePath = "$chapterDir/$fileName" + ImageResponse.saveImage(filePath, image) + withContext(Dispatchers.IO) { + image.close() + } + } finally { + // always cancel the page progress job even if it throws an exception to avoid memory leaks + pageProgressJob?.cancel() + } + // TODO: retry on error with 2,4,8 seconds of wait + download.progress = ((pageNum + 1).toFloat()) / pageCount + step(download, false) + } return true } From 50e29562821b5c2ddd90584284cef89249aca5ac Mon Sep 17 00:00:00 2001 From: akabhirav Date: Fri, 10 Feb 2023 18:15:42 +0530 Subject: [PATCH 5/6] optimize and handle partial downloads --- .../manga/impl/download/FolderProvider.kt | 49 +++++++++++-------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt index fac46f711..e7d5f0a6c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt @@ -1,14 +1,14 @@ package suwayomi.tachidesk.manga.impl.download +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.FlowPreview +import kotlinx.coroutines.Job import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.withContext -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Job import suwayomi.tachidesk.manga.impl.Page import suwayomi.tachidesk.manga.impl.Page.getPageName import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter @@ -39,28 +39,30 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man step: KSuspendFunction2 ): Boolean { val pageCount = download.chapter.pageCount + val chapterDir = getChapterDir(mangaId, chapterId) + val folder = File(chapterDir) + folder.mkdirs() + for (pageNum in 0 until pageCount) { var pageProgressJob: Job? = null + val fileName = getPageName(pageNum) // might have to change this to index stored in database + if (isExistingFile(folder, fileName)) continue try { val image = Page.getPageImage( mangaId = download.mangaId, chapterIndex = download.chapterIndex, - index = pageNum, - progressFlow = { flow -> - pageProgressJob = flow - .sample(100) - .distinctUntilChanged() - .onEach { - download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount - step(null, false) // don't throw on canceled download here since we can't do anything - } - .launchIn(scope) - } - ).first - val chapterDir = getChapterDir(mangaId, chapterId) - val folder = File(chapterDir) - folder.mkdirs() - val fileName = getPageName(pageNum) // might have to change this to index stored in database + index = pageNum + ) { flow -> + pageProgressJob = flow + .sample(100) + .distinctUntilChanged() + .onEach { + download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount + step(null, false) // don't throw on canceled download here since we can't do anything + } + .launchIn(scope) + }.first + val filePath = "$chapterDir/$fileName" ImageResponse.saveImage(filePath, image) withContext(Dispatchers.IO) { @@ -81,4 +83,11 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man val chapterDir = getChapterDir(mangaId, chapterId) return File(chapterDir).deleteRecursively() } + + private fun isExistingFile(folder: File, fileName: String): Boolean { + val existingFile = folder.listFiles { file -> + file.isFile && file.name.startsWith(fileName) + }?.firstOrNull() + return existingFile?.exists() == true + } } From 02268312865d737579e0e08649cbed7230973d39 Mon Sep 17 00:00:00 2001 From: akabhirav Date: Sat, 11 Feb 2023 16:13:38 +0530 Subject: [PATCH 6/6] made code review changes --- .../manga/impl/ChapterDownloadHelper.kt | 3 +-- .../impl/download/DownloadedFilesProvider.kt | 3 +-- .../manga/impl/download/FolderProvider.kt | 16 +++++----------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt index a80c37228..768e80e01 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt @@ -5,7 +5,6 @@ import suwayomi.tachidesk.manga.impl.download.DownloadedFilesProvider import suwayomi.tachidesk.manga.impl.download.FolderProvider import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import java.io.InputStream -import kotlin.reflect.KSuspendFunction2 object ChapterDownloadHelper { fun getImage(mangaId: Int, chapterId: Int, index: Int): Pair { @@ -21,7 +20,7 @@ object ChapterDownloadHelper { chapterId: Int, download: DownloadChapter, scope: CoroutineScope, - step: KSuspendFunction2 + step: suspend (DownloadChapter?, Boolean) -> Unit ): Boolean { return provider(mangaId, chapterId).download(download, scope, step) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt index 7f40f7ed4..47c7db610 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/DownloadedFilesProvider.kt @@ -3,7 +3,6 @@ package suwayomi.tachidesk.manga.impl.download import kotlinx.coroutines.CoroutineScope import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter import java.io.InputStream -import kotlin.reflect.KSuspendFunction2 /* * Base class for downloaded chapter files provider, example: Folder, Archive @@ -14,7 +13,7 @@ abstract class DownloadedFilesProvider(val mangaId: Int, val chapterId: Int) { abstract suspend fun download( download: DownloadChapter, scope: CoroutineScope, - step: KSuspendFunction2 + step: suspend (DownloadChapter?, Boolean) -> Unit ): Boolean abstract fun delete(): Boolean diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt index e7d5f0a6c..84176eccd 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/FolderProvider.kt @@ -1,14 +1,12 @@ package suwayomi.tachidesk.manga.impl.download import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.FlowPreview import kotlinx.coroutines.Job import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.launchIn import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.sample -import kotlinx.coroutines.withContext import suwayomi.tachidesk.manga.impl.Page import suwayomi.tachidesk.manga.impl.Page.getPageName import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter @@ -17,7 +15,6 @@ import suwayomi.tachidesk.manga.impl.util.storage.ImageResponse import java.io.File import java.io.FileInputStream import java.io.InputStream -import kotlin.reflect.KSuspendFunction2 /* * Provides downloaded files when pages were downloaded into folders @@ -36,7 +33,7 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man override suspend fun download( download: DownloadChapter, scope: CoroutineScope, - step: KSuspendFunction2 + step: suspend (DownloadChapter?, Boolean) -> Unit ): Boolean { val pageCount = download.chapter.pageCount val chapterDir = getChapterDir(mangaId, chapterId) @@ -48,7 +45,7 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man val fileName = getPageName(pageNum) // might have to change this to index stored in database if (isExistingFile(folder, fileName)) continue try { - val image = Page.getPageImage( + Page.getPageImage( mangaId = download.mangaId, chapterIndex = download.chapterIndex, index = pageNum @@ -61,12 +58,9 @@ class FolderProvider(mangaId: Int, chapterId: Int) : DownloadedFilesProvider(man step(null, false) // don't throw on canceled download here since we can't do anything } .launchIn(scope) - }.first - - val filePath = "$chapterDir/$fileName" - ImageResponse.saveImage(filePath, image) - withContext(Dispatchers.IO) { - image.close() + }.first.use { image -> + val filePath = "$chapterDir/$fileName" + ImageResponse.saveImage(filePath, image) } } finally { // always cancel the page progress job even if it throws an exception to avoid memory leaks