Add support for opds-pse for undownloaded chapters (#1278)

* Add OPDS page streaming for undownloaded chapters

* Add [D] in chapter title prefix when isDownloaded

* Removed Chapter.isDownloaded check in query for other opds endpoints

* Add chapter progression tracking for streaming and refactor code

* dd ❌ Unicode for chapters with 0 pages [post pageRefresh]

* Add Library Updates feed and remove redundant metadata fetching for OPDS chapters and manga

* Address PR comments & add chapter markAsRead for cbzDownload

* Address PR comment/s

* Rem. markAsRead for chapter download

* Rem. markAsRead for chapter download

---------

Co-authored-by: ShowY <showypro@gmail.com>

server/src/main/kotlin/suwayomi/tachidesk/manga/controller/MangaController.kt

@@ -401,6 +401,7 @@ object MangaController {
             pathParam<Int>("mangaId"),
             pathParam<Int>("chapterIndex"),
             pathParam<Int>("index"),
+            queryParam<Boolean?>("updateProgress"),
             documentWith = {
                 withOperation {
                     summary("Get a chapter page")
@@ -409,14 +410,18 @@ object MangaController {
                     )
                 }
             },
-            behaviorOf = { ctx, mangaId, chapterIndex, index ->
+            behaviorOf = { ctx, mangaId, chapterIndex, index, updateProgress ->
                 ctx.future {
-                    future { Page.getPageImage(mangaId, chapterIndex, index) }
+                    future { Page.getPageImage(mangaId, chapterIndex, index, null) }
                         .thenApply {
                             ctx.header("content-type", it.second)
                             val httpCacheSeconds = 1.days.inWholeSeconds
                             ctx.header("cache-control", "max-age=$httpCacheSeconds")
                             ctx.result(it.first)
+
+                            if (updateProgress == true) {
+                                Chapter.updateChapterProgress(mangaId, chapterIndex, pageNo = index)
+                            }
                         }
                 }
             },
@@ -437,10 +442,11 @@ object MangaController {
             },
             behaviorOf = { ctx, chapterId ->
                 ctx.future {
-                    future { ChapterDownloadHelper.getCbzDownload(chapterId) }
-                        .thenApply { (inputStream, contentType, fileName) ->
-                            ctx.header("Content-Type", contentType)
+                    future { ChapterDownloadHelper.getCbzForDownload(chapterId) }
+                        .thenApply { (inputStream, fileName, fileSize) ->
+                            ctx.header("Content-Type", "application/vnd.comicbook+zip")
                             ctx.header("Content-Disposition", "attachment; filename=\"$fileName\"")
+                            ctx.header("Content-Length", fileSize.toString())
                             ctx.result(inputStream)
                         }
                 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -262,8 +262,8 @@ object Chapter {
                 // we got some clean up due
                 if (chaptersIdsToDelete.isNotEmpty()) {
                     transaction {
-                        PageTable.deleteWhere { PageTable.chapter inList chaptersIdsToDelete }
-                        ChapterTable.deleteWhere { ChapterTable.id inList chaptersIdsToDelete }
+                        PageTable.deleteWhere { chapter inList chaptersIdsToDelete }
+                        ChapterTable.deleteWhere { id inList chaptersIdsToDelete }
                     }
                 }
 
@@ -321,7 +321,7 @@ object Chapter {
                     }
 
                     MangaTable.update({ MangaTable.id eq mangaId }) {
-                        it[MangaTable.chaptersLastFetchedAt] = Instant.now().epochSecond
+                        it[chaptersLastFetchedAt] = Instant.now().epochSecond
                     }
                 }
 
@@ -443,7 +443,7 @@ object Chapter {
                     }
                     lastPageRead?.also {
                         update[ChapterTable.lastPageRead] = it
-                        update[ChapterTable.lastReadAt] = Instant.now().epochSecond
+                        update[lastReadAt] = Instant.now().epochSecond
                     }
                 }
             }
@@ -534,7 +534,7 @@ object Chapter {
                 }
                 lastPageRead?.also {
                     update[ChapterTable.lastPageRead] = it
-                    update[ChapterTable.lastReadAt] = now
+                    update[lastReadAt] = now
                 }
             }
         }
@@ -603,7 +603,7 @@ object Chapter {
                 ChapterMetaTable.insert {
                     it[ChapterMetaTable.key] = key
                     it[ChapterMetaTable.value] = value
-                    it[ChapterMetaTable.ref] = chapterId
+                    it[ref] = chapterId
                 }
             } else {
                 ChapterMetaTable.update({ (ChapterMetaTable.ref eq chapterId) and (ChapterMetaTable.key eq key) }) {
@@ -693,4 +693,33 @@ object Chapter {
                     }
             }
         }
+
+    fun updateChapterProgress(
+        mangaId: Int,
+        chapterIndex: Int,
+        pageNo: Int,
+    ) {
+        val chapterData =
+            transaction {
+                ChapterTable
+                    .selectAll()
+                    .where {
+                        (ChapterTable.sourceOrder eq chapterIndex) and
+                            (ChapterTable.manga eq mangaId)
+                    }.first()
+                    .let { ChapterTable.toDataClass(it) }
+            }
+
+        val oneIndexedPageNo = pageNo.inc()
+        val isRead = chapterData.pageCount.takeIf { it == oneIndexedPageNo }?.let { true }
+
+        modifyChapter(
+            mangaId,
+            chapterIndex,
+            isRead = isRead,
+            lastPageRead = pageNo,
+            isBookmarked = null,
+            markPrevRead = null,
+        )
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt

@@ -13,7 +13,6 @@ import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
 import suwayomi.tachidesk.server.serverConfig
 import java.io.File
-import java.io.IOException
 import java.io.InputStream
 
 object ChapterDownloadHelper {
@@ -48,26 +47,28 @@ object ChapterDownloadHelper {
         return FolderProvider(mangaId, chapterId)
     }
 
-    suspend fun getCbzDownload(chapterId: Int): Triple<InputStream, String, String> {
+    fun getArchiveStreamWithSize(
+        mangaId: Int,
+        chapterId: Int,
+    ): Pair<InputStream, Long> = provider(mangaId, chapterId).getAsArchiveStream()
+
+    fun getCbzForDownload(chapterId: Int): Triple<InputStream, String, Long> {
         val (chapterData, mangaTitle) =
             transaction {
                 val row =
                     (ChapterTable innerJoin MangaTable)
                         .select(ChapterTable.columns + MangaTable.columns)
                         .where { ChapterTable.id eq chapterId }
-                        .firstOrNull() ?: throw Exception("Chapter not found")
+                        .firstOrNull() ?: throw IllegalArgumentException("ChapterId $chapterId not found")
                 val chapter = ChapterTable.toDataClass(row)
                 val title = row[MangaTable.title]
                 Pair(chapter, title)
             }
 
-        val provider = provider(chapterData.mangaId, chapterData.id)
-        return if (provider is ArchiveProvider) {
-            val cbzFile = File(getChapterCbzPath(chapterData.mangaId, chapterData.id))
-            val fileName = "$mangaTitle - [${chapterData.scanlator}] ${chapterData.name}.cbz"
-            Triple(cbzFile.inputStream(), "application/vnd.comicbook+zip", fileName)
-        } else {
-            throw IOException("Chapter not available as CBZ")
-        }
+        val fileName = "$mangaTitle - [${chapterData.scanlator}] ${chapterData.name}.cbz"
+
+        val cbzFile = provider(chapterData.mangaId, chapterData.id).getAsArchiveStream()
+
+        return Triple(cbzFile.first, fileName, cbzFile.second)
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt

@@ -33,7 +33,6 @@ suspend fun getChapterDownloadReady(
     mangaId: Int? = null,
 ): ChapterDataClass {
     val chapter = ChapterForDownload(chapterId, chapterIndex, mangaId)
-
     return chapter.asDownloadReady()
 }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt

@@ -125,7 +125,10 @@ abstract class ChaptersFilesProvider<Type : FileType>(
                                 .distinctUntilChanged()
                                 .onEach {
                                     download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
-                                    step(null, false) // don't throw on canceled download here since we can't do anything
+                                    step(
+                                        null,
+                                        false,
+                                    ) // don't throw on canceled download here since we can't do anything
                                 }.launchIn(scope)
                     }.first
                     .close()
@@ -159,4 +162,6 @@ abstract class ChaptersFilesProvider<Type : FileType>(
         FileDownload3Args(::downloadImpl)
 
     abstract override fun delete(): Boolean
+
+    abstract fun getAsArchiveStream(): Pair<InputStream, Long>
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/ArchiveProvider.kt

@@ -88,6 +88,15 @@ class ArchiveProvider(
         return cbzDeleted
     }
 
+    override fun getAsArchiveStream(): Pair<InputStream, Long> {
+        val cbzFile =
+            File(getChapterCbzPath(mangaId, chapterId))
+                .takeIf { it.exists() }
+                ?: throw IllegalArgumentException("CBZ file not found for chapter ID: $chapterId (Manga ID: $mangaId)")
+
+        return cbzFile.inputStream() to cbzFile.length()
+    }
+
     private fun extractCbzFile(
         cbzFile: File,
         chapterFolder: File,

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/impl/FolderProvider.kt

@@ -7,8 +7,14 @@ import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
 import suwayomi.tachidesk.manga.impl.util.storage.FileDeletionHelper
 import suwayomi.tachidesk.server.ApplicationDirs
 import uy.kohesive.injekt.injectLazy
+import java.io.BufferedOutputStream
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.FileInputStream
+import java.io.InputStream
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
 
 private val applicationDirs: ApplicationDirs by injectLazy()
 
@@ -58,4 +64,31 @@ class FolderProvider(
         FileDeletionHelper.cleanupParentFoldersFor(chapterDir, applicationDirs.mangaDownloadsRoot)
         return chapterDirDeleted
     }
+
+    override fun getAsArchiveStream(): Pair<InputStream, Long> {
+        val chapterDir = File(getChapterDownloadPath(mangaId, chapterId))
+
+        if (!chapterDir.exists() || !chapterDir.isDirectory || chapterDir.listFiles().isNullOrEmpty()) {
+            throw IllegalArgumentException("Invalid folder to create CBZ for chapter ID: $chapterId")
+        }
+
+        val byteArrayOutputStream = ByteArrayOutputStream()
+        ZipOutputStream(BufferedOutputStream(byteArrayOutputStream)).use { zipOutputStream ->
+            chapterDir
+                .listFiles()
+                ?.filter { it.isFile }
+                ?.sortedBy { it.name }
+                ?.forEach { imageFile ->
+                    FileInputStream(imageFile).use { fileInputStream ->
+                        val zipEntry = ZipEntry(imageFile.name)
+                        zipOutputStream.putNextEntry(zipEntry)
+                        fileInputStream.copyTo(zipOutputStream)
+                        zipOutputStream.closeEntry()
+                    }
+                }
+        }
+
+        val zipData = byteArrayOutputStream.toByteArray()
+        return ByteArrayInputStream(zipData) to zipData.size.toLong()
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt

@@ -14,7 +14,6 @@ import org.jetbrains.exposed.sql.selectAll
 import org.jetbrains.exposed.sql.transactions.transaction
 import suwayomi.tachidesk.manga.impl.Chapter.getChapterMetaMap
 import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
-import suwayomi.tachidesk.manga.model.table.MangaTable.nullable
 
 object ChapterTable : IntIdTable() {
     val url = varchar("url", 2048)
@@ -41,31 +40,43 @@ object ChapterTable : IntIdTable() {
     val manga = reference("manga", MangaTable, ReferenceOption.CASCADE)
 }
 
-fun ChapterTable.toDataClass(chapterEntry: ResultRow) =
-    ChapterDataClass(
-        id = chapterEntry[id].value,
-        url = chapterEntry[url],
-        name = chapterEntry[name],
-        uploadDate = chapterEntry[date_upload],
-        chapterNumber = chapterEntry[chapter_number],
-        scanlator = chapterEntry[scanlator],
-        mangaId = chapterEntry[manga].value,
-        read = chapterEntry[isRead],
-        bookmarked = chapterEntry[isBookmarked],
-        lastPageRead = chapterEntry[lastPageRead],
-        lastReadAt = chapterEntry[lastReadAt],
-        index = chapterEntry[sourceOrder],
-        fetchedAt = chapterEntry[fetchedAt],
-        realUrl = chapterEntry[realUrl],
-        downloaded = chapterEntry[isDownloaded],
-        pageCount = chapterEntry[pageCount],
-        chapterCount =
+fun ChapterTable.toDataClass(
+    chapterEntry: ResultRow,
+    includeChapterCount: Boolean = true,
+    includeChapterMeta: Boolean = true,
+) = ChapterDataClass(
+    id = chapterEntry[id].value,
+    url = chapterEntry[url],
+    name = chapterEntry[name],
+    uploadDate = chapterEntry[date_upload],
+    chapterNumber = chapterEntry[chapter_number],
+    scanlator = chapterEntry[scanlator],
+    mangaId = chapterEntry[manga].value,
+    read = chapterEntry[isRead],
+    bookmarked = chapterEntry[isBookmarked],
+    lastPageRead = chapterEntry[lastPageRead],
+    lastReadAt = chapterEntry[lastReadAt],
+    index = chapterEntry[sourceOrder],
+    fetchedAt = chapterEntry[fetchedAt],
+    realUrl = chapterEntry[realUrl],
+    downloaded = chapterEntry[isDownloaded],
+    pageCount = chapterEntry[pageCount],
+    chapterCount =
+        if (includeChapterCount) {
             transaction {
                 ChapterTable
                     .selectAll()
                     .where { manga eq chapterEntry[manga].value }
                     .count()
                     .toInt()
-            },
-        meta = getChapterMetaMap(chapterEntry[id]),
-    )
+            }
+        } else {
+            null
+        },
+    meta =
+        if (includeChapterMeta) {
+            getChapterMetaMap(chapterEntry[id])
+        } else {
+            emptyMap()
+        },
+)

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt

@@ -46,28 +46,35 @@ object MangaTable : IntIdTable() {
     val updateStrategy = varchar("update_strategy", 256).default(UpdateStrategy.ALWAYS_UPDATE.name)
 }
 
-fun MangaTable.toDataClass(mangaEntry: ResultRow) =
-    MangaDataClass(
-        id = mangaEntry[this.id].value,
-        sourceId = mangaEntry[sourceReference].toString(),
-        url = mangaEntry[url],
-        title = mangaEntry[title],
-        thumbnailUrl = proxyThumbnailUrl(mangaEntry[this.id].value),
-        thumbnailUrlLastFetched = mangaEntry[thumbnailUrlLastFetched],
-        initialized = mangaEntry[initialized],
-        artist = mangaEntry[artist],
-        author = mangaEntry[author],
-        description = mangaEntry[description],
-        genre = mangaEntry[genre].toGenreList(),
-        status = Companion.valueOf(mangaEntry[status]).name,
-        inLibrary = mangaEntry[inLibrary],
-        inLibraryAt = mangaEntry[inLibraryAt],
-        meta = getMangaMetaMap(mangaEntry[id].value),
-        realUrl = mangaEntry[realUrl],
-        lastFetchedAt = mangaEntry[lastFetchedAt],
-        chaptersLastFetchedAt = mangaEntry[chaptersLastFetchedAt],
-        updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy]),
-    )
+fun MangaTable.toDataClass(
+    mangaEntry: ResultRow,
+    includeMangaMeta: Boolean = true,
+) = MangaDataClass(
+    id = mangaEntry[this.id].value,
+    sourceId = mangaEntry[sourceReference].toString(),
+    url = mangaEntry[url],
+    title = mangaEntry[title],
+    thumbnailUrl = proxyThumbnailUrl(mangaEntry[this.id].value),
+    thumbnailUrlLastFetched = mangaEntry[thumbnailUrlLastFetched],
+    initialized = mangaEntry[initialized],
+    artist = mangaEntry[artist],
+    author = mangaEntry[author],
+    description = mangaEntry[description],
+    genre = mangaEntry[genre].toGenreList(),
+    status = Companion.valueOf(mangaEntry[status]).name,
+    inLibrary = mangaEntry[inLibrary],
+    inLibraryAt = mangaEntry[inLibraryAt],
+    meta =
+        if (includeMangaMeta) {
+            getMangaMetaMap(mangaEntry[id].value)
+        } else {
+            emptyMap()
+        },
+    realUrl = mangaEntry[realUrl],
+    lastFetchedAt = mangaEntry[lastFetchedAt],
+    chaptersLastFetchedAt = mangaEntry[chaptersLastFetchedAt],
+    updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy]),
+)
 
 enum class MangaStatus(
     val value: Int,