Improve Downloads Handling (#1387)

* Improve Downloads Handling

* Update known pagecount for downloaded chapters

* Get fresh data for downloadReady

* Format

* Assume downloaded if first page is found

* Filter out ComicInfoFile

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt

@@ -22,6 +22,11 @@ object ChapterDownloadHelper {
         index: Int,
     ): Pair<InputStream, String> = provider(mangaId, chapterId).getImage().execute(index)
 
+    fun getImageCount(
+        mangaId: Int,
+        chapterId: Int,
+    ): Int = provider(mangaId, chapterId).getImageCount()
+
     fun delete(
         mangaId: Int,
         chapterId: Int,

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt

@@ -110,7 +110,7 @@ object Page {
             }
         }
 
-        val fileName = getPageName(index)
+        val fileName = getPageName(index, chapterEntry[ChapterTable.pageCount])
 
         val cacheSaveDir = getChapterCachePath(mangaId, chapterId)
 
@@ -121,5 +121,8 @@ object Page {
     }
 
     /** converts 0 to "001" */
-    fun getPageName(index: Int): String = String.format("%03d", index + 1)
+    fun getPageName(
+        index: Int,
+        pageCount: Int,
+    ): String = String.format("%0${pageCount.toString().length.coerceAtLeast(3)}d", index + 1)
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt

@@ -70,7 +70,7 @@ private class ChapterForDownload(
 
         val isMarkedAsDownloaded = chapterEntry[ChapterTable.isDownloaded]
         val doesFirstPageExist = firstPageExists()
-        val isDownloaded = isMarkedAsDownloaded && doesFirstPageExist
+        val isDownloaded = isMarkedAsDownloaded || doesFirstPageExist
 
         log.debug { "isDownloaded= $isDownloaded (isMarkedAsDownloaded= $isMarkedAsDownloaded, doesFirstPageExist= $doesFirstPageExist)" }
 
@@ -80,12 +80,22 @@ private class ChapterForDownload(
             markAsNotDownloaded()
 
             updatePageList()
+        } else {
+            updatePageCount(ChapterDownloadHelper.getImageCount(mangaId, chapterId))
         }
 
         return asDataClass()
     }
 
-    private fun asDataClass() = ChapterTable.toDataClass(chapterEntry)
+    private fun asDataClass() =
+        ChapterTable.toDataClass(
+            transaction {
+                ChapterTable
+                    .selectAll()
+                    .where { ChapterTable.id eq chapterId }
+                    .first()
+            },
+        )
 
     init {
         chapterEntry = freshChapterEntry(optChapterId, optChapterIndex, optMangaId)
@@ -160,19 +170,15 @@ private class ChapterForDownload(
             }
         }
 
-        updatePageCount(pageList, chapterId)
+        updatePageCount(pageList.size)
 
         // chapter was updated
         chapterEntry = freshChapterEntry(chapterId, chapterIndex, mangaId)
     }
 
-    private fun updatePageCount(
+    private fun updatePageCount(pageCount: Int) {
-        pageList: List<Page>,
-        chapterId: Int,
-    ) {
         transaction {
             ChapterTable.update({ ChapterTable.id eq chapterId }) {
-                val pageCount = pageList.size
                 it[ChapterTable.pageCount] = pageCount
                 it[ChapterTable.lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(pageCount - 1).coerceAtLeast(0)
             }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt

@@ -9,8 +9,10 @@ import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.flow.sample
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
 import org.jetbrains.exposed.sql.selectAll
 import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Page
 import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
 import suwayomi.tachidesk.manga.impl.util.createComicInfoFile
@@ -76,6 +78,8 @@ abstract class ChaptersFilesProvider<Type : FileType>(
         return Pair(getImageInputStream(image).buffered(), "image/$imageFileType")
     }
 
+    fun getImageCount(): Int = getImageFiles().filter { it.getName() != COMIC_INFO_FILE }.size
+
     override fun getImage(): RetrieveFile1Args<Int> = RetrieveFile1Args(::getImageImpl)
 
     /**
@@ -100,45 +104,56 @@ abstract class ChaptersFilesProvider<Type : FileType>(
         downloadCacheFolder.mkdirs()
 
         val pageCount = download.chapter.pageCount
-        for (pageNum in 0 until pageCount) {
+        if (
-            var pageProgressJob: Job? = null
+            downloadCacheFolder
-            val fileName = Page.getPageName(pageNum) // might have to change this to index stored in database
+                .listFiles()
-
+                .orEmpty()
-            val pageExistsInFinalDownloadFolder = ImageResponse.findFileNameStartingWith(finalDownloadFolder, fileName) != null
+                .filter { it.name != COMIC_INFO_FILE }
-            val pageExistsInCacheDownloadFolder = ImageResponse.findFileNameStartingWith(cacheChapterDir, fileName) != null
+                .size >= pageCount
-
+        ) {
-            val doesPageAlreadyExist = pageExistsInFinalDownloadFolder || pageExistsInCacheDownloadFolder
+            download.progress = 1f
-            if (doesPageAlreadyExist) {
-                continue
-            }
-
-            try {
-                Page
-                    .getPageImage(
-                        mangaId = download.mangaId,
-                        chapterIndex = download.chapterIndex,
-                        index = pageNum,
-                    ) { flow ->
-                        pageProgressJob =
-                            flow
-                                .sample(100)
-                                .distinctUntilChanged()
-                                .onEach {
-                                    download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
-                                    step(
-                                        null,
-                                        false,
-                                    ) // don't throw on canceled download here since we can't do anything
-                                }.launchIn(scope)
-                    }.first
-                    .close()
-            } finally {
-                // always cancel the page progress job even if it throws an exception to avoid memory leaks
-                pageProgressJob?.cancel()
-            }
-            // TODO: retry on error with 2,4,8 seconds of wait
-            download.progress = ((pageNum + 1).toFloat()) / pageCount
             step(download, false)
+        } else {
+            for (pageNum in 0 until pageCount) {
+                var pageProgressJob: Job? = null
+                val fileName = Page.getPageName(pageNum, pageCount) // might have to change this to index stored in database
+
+                val pageExistsInFinalDownloadFolder = ImageResponse.findFileNameStartingWith(finalDownloadFolder, fileName) != null
+                val pageExistsInCacheDownloadFolder = ImageResponse.findFileNameStartingWith(cacheChapterDir, fileName) != null
+
+                val doesPageAlreadyExist = pageExistsInFinalDownloadFolder || pageExistsInCacheDownloadFolder
+                if (doesPageAlreadyExist) {
+                    continue
+                }
+
+                try {
+                    Page
+                        .getPageImage(
+                            mangaId = download.mangaId,
+                            chapterIndex = download.chapterIndex,
+                            index = pageNum,
+                        ) { flow ->
+                            pageProgressJob =
+                                flow
+                                    .sample(100)
+                                    .distinctUntilChanged()
+                                    .onEach {
+                                        download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
+                                        step(
+                                            null,
+                                            false,
+                                        ) // don't throw on canceled download here since we can't do anything
+                                    }.launchIn(scope)
+                        }.first
+                        .close()
+                } finally {
+                    // always cancel the page progress job even if it throws an exception to avoid memory leaks
+                    pageProgressJob?.cancel()
+                }
+                // TODO: retry on error with 2,4,8 seconds of wait
+                download.progress = ((pageNum + 1).toFloat()) / pageCount
+                step(download, false)
+            }
         }
 
         createComicInfoFile(
@@ -153,6 +168,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
 
         handleSuccessfulDownload()
 
+        transaction {
+            ChapterTable.update({ ChapterTable.id eq chapterId }) {
+                it[ChapterTable.pageCount] = getImageCount()
+            }
+        }
+
         File(cacheChapterDir).deleteRecursively()
 
         return true

server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt

@@ -15,16 +15,18 @@ import kotlin.test.assertEquals
 class PageTest : ApplicationTest() {
     @Test
     fun testGetPageName() {
-        val tests = listOf(0, 1, 2, 100)
+        val tests = listOf(0 to 5, 1 to 5, 2 to 5, 100 to 100, 998 to 1000, 1400 to 1500)
 
         val testResults =
-            tests.map {
+            tests.map { (page, count) ->
-                getPageName(it)
+                getPageName(page, count)
             }
 
         assertEquals(testResults[0], "001")
         assertEquals(testResults[1], "002")
         assertEquals(testResults[2], "003")
         assertEquals(testResults[3], "101")
+        assertEquals(testResults[4], "0999")
+        assertEquals(testResults[5], "1401")
     }
 }

server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt

@@ -45,10 +45,16 @@ class TestUpdater : IUpdater {
 
     override val status: Flow<UpdateStatus>
         get() = TODO("Not yet implemented")
+    override val updates: Flow<UpdateUpdates>
+        get() = TODO("Not yet implemented")
     override val statusDeprecated: StateFlow<UpdateStatus>
         get() = TODO("Not yet implemented")
 
     override fun reset() {
         TODO("Not yet implemented")
     }
+
+    override fun getStatus(): UpdateUpdates {
+        TODO("Not yet implemented")
+    }
 }