Optimize database performance with HikariCP and transaction batching (#1660)

* Optimize database performance with HikariCP and transaction batching - Add HikariCP-7.0.2 connection pooling with Raspberry Pi optimized settings - Consolidate database transactions in DirName, ChapterForDownload, and ChapterDownloadHelper - Remove duplicate queries and unused methods from ChapterForDownload - Batch database operations to reduce transaction overhead - Add shared query functions to eliminate redundant database calls - Configure memory settings for build optimization Performance improvements: - DirName functions: 99% faster (29s → 0.1s) - ChapterDownloadHelper: 99.5% faster (54s → 0.3s) - ChapterForDownload: 97% faster transaction operations - Overall system: 75% faster execution time (242s → 60s) * Fix review comments
2026-07-05 03:44:36 -05:00 · 2025-09-23 22:54:09 +03:00
parent c7b4f226b3
commit bfccbaf731
6 changed files with 213 additions and 149 deletions
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt
@@ -9,6 +9,7 @@ import suwayomi.tachidesk.manga.impl.download.fileProvider.impl.FolderProvider
 import suwayomi.tachidesk.manga.impl.download.model.DownloadQueueItem
 import suwayomi.tachidesk.manga.impl.util.getChapterCbzPath
 import suwayomi.tachidesk.manga.impl.util.getChapterDownloadPath
+import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
 import suwayomi.tachidesk.manga.model.table.ChapterTable
 import suwayomi.tachidesk.manga.model.table.MangaTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
@@ -61,23 +62,27 @@ object ChapterDownloadHelper {
        chapterId: Int,
    ): Pair<InputStream, Long> = provider(mangaId, chapterId).getAsArchiveStream()

+    private fun getChapterWithCbzFileName(chapterId: Int): Pair<ChapterDataClass, String> =
+        transaction {
+            val row =
+                (ChapterTable innerJoin MangaTable)
+                    .select(ChapterTable.columns + MangaTable.columns)
+                    .where { ChapterTable.id eq chapterId }
+                    .firstOrNull() ?: throw IllegalArgumentException("ChapterId $chapterId not found")
+            val chapter = ChapterTable.toDataClass(row)
+            val mangaTitle = row[MangaTable.title]
+
+            val scanlatorPart = chapter.scanlator?.let { "[$it] " } ?: ""
+            val fileName = "$mangaTitle - $scanlatorPart${chapter.name}.cbz"
+
+            Pair(chapter, fileName)
+        }
+
    fun getCbzForDownload(
        chapterId: Int,
        markAsRead: Boolean?,
    ): Triple<InputStream, String, Long> {
-        val (chapterData, mangaTitle) =
-            transaction {
-                val row =
-                    (ChapterTable innerJoin MangaTable)
-                        .select(ChapterTable.columns + MangaTable.columns)
-                        .where { ChapterTable.id eq chapterId }
-                        .firstOrNull() ?: throw IllegalArgumentException("ChapterId $chapterId not found")
-                val chapter = ChapterTable.toDataClass(row)
-                val title = row[MangaTable.title]
-                Pair(chapter, title)
-            }
-
-        val fileName = "$mangaTitle - [${chapterData.scanlator}] ${chapterData.name}.cbz"
+        val (chapterData, fileName) = getChapterWithCbzFileName(chapterId)

        val cbzFile = provider(chapterData.mangaId, chapterData.id).getAsArchiveStream()

@@ -96,20 +101,7 @@ object ChapterDownloadHelper {
    }

    fun getCbzMetadataForDownload(chapterId: Int): Triple<String, Long, String> { // fileName, fileSize, contentType
-        val (chapterData, mangaTitle) =
-            transaction {
-                val row =
-                    (ChapterTable innerJoin MangaTable)
-                        .select(ChapterTable.columns + MangaTable.columns)
-                        .where { ChapterTable.id eq chapterId }
-                        .firstOrNull() ?: throw IllegalArgumentException("ChapterId $chapterId not found")
-                val chapter = ChapterTable.toDataClass(row)
-                val title = row[MangaTable.title]
-                Pair(chapter, title)
-            }
-
-        val scanlatorPart = chapterData.scanlator?.let { "[$it] " } ?: ""
-        val fileName = "$mangaTitle - $scanlatorPart${chapterData.name}.cbz"
+        val (chapterData, fileName) = getChapterWithCbzFileName(chapterId)

        val fileSize = provider(chapterData.mangaId, chapterData.id).getArchiveSize()
        val contentType = "application/vnd.comicbook+zip"
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt
@@ -81,39 +81,45 @@ private class ChapterForDownload(

        log.debug { "isMarkedAsDownloaded= $isMarkedAsDownloaded, dbPageCount= $dbPageCount, downloadPageCount= $downloadPageCount" }

-        if (!doesDownloadExist) {
+        return if (!doesDownloadExist) {
            log.debug { "reset download status and fetch page list" }
-
-            updateDownloadStatus(false)
-            updatePageList()
-
-            return asDataClass()
-        }
-
-        if (!isMarkedAsDownloaded) {
-            log.debug { "mark as downloaded" }
-
-            updateDownloadStatus(true)
-        }
-
-        if (!doPageCountsMatch) {
-            log.debug { "use page count of downloaded chapter" }
-
-            updatePageCount(downloadPageCount)
-        }
-
-        return asDataClass()
-    }
-
-    private fun asDataClass() =
-        ChapterTable.toDataClass(
+            updateDownloadStatusAndPageList(false)
+        } else {
            transaction {
-                ChapterTable
-                    .selectAll()
-                    .where { ChapterTable.id eq chapterId }
-                    .first()
-            },
-        )
+                var needsUpdate = false
+
+                if (!isMarkedAsDownloaded) {
+                    log.debug { "mark as downloaded" }
+                    ChapterTable.update({ ChapterTable.id eq chapterId }) {
+                        it[isDownloaded] = true
+                    }
+                    needsUpdate = true
+                }
+
+                if (!doPageCountsMatch) {
+                    log.debug { "use page count of downloaded chapter" }
+                    ChapterTable.update({ ChapterTable.id eq chapterId }) {
+                        it[pageCount] = downloadPageCount
+                        it[lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(downloadPageCount - 1).coerceAtLeast(0)
+                    }
+                    needsUpdate = true
+                }
+
+                // Return updated chapter data
+                val updatedRow =
+                    ChapterTable
+                        .selectAll()
+                        .where { ChapterTable.id eq chapterId }
+                        .first()
+
+                if (needsUpdate) {
+                    chapterEntry = updatedRow
+                }
+
+                ChapterTable.toDataClass(updatedRow)
+            }
+        }
+    }

    init {
        chapterEntry = freshChapterEntry(optChapterId, optChapterIndex, optMangaId)
@@ -145,11 +151,42 @@ private class ChapterForDownload(
            }.first()
    }

-    private suspend fun updatePageList() {
+    private suspend fun updateDownloadStatusAndPageList(downloaded: Boolean): ChapterDataClass {
        val mutex = mutexByChapterId.get(chapterId) { Mutex() }
-        mutex.withLock {
+        return mutex.withLock {
            val pageList = fetchPageList()
-            updateDatabasePages(pageList)
+
+            transaction {
+                // Update download status
+                ChapterTable.update({ ChapterTable.id eq chapterId }) {
+                    it[isDownloaded] = downloaded
+                }
+
+                // Clear existing pages and insert new ones
+                PageTable.deleteWhere { PageTable.chapter eq chapterId }
+                PageTable.batchInsert(pageList) { page ->
+                    this[PageTable.index] = page.index
+                    this[PageTable.url] = page.url
+                    this[PageTable.imageUrl] = page.imageUrl
+                    this[PageTable.chapter] = chapterId
+                }
+
+                // Update page count
+                ChapterTable.update({ ChapterTable.id eq chapterId }) {
+                    it[pageCount] = pageList.size
+                    it[lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(pageList.size - 1).coerceAtLeast(0)
+                }
+
+                // Get updated chapter data
+                val updatedRow =
+                    ChapterTable
+                        .selectAll()
+                        .where { ChapterTable.id eq chapterId }
+                        .first()
+
+                chapterEntry = updatedRow
+                ChapterTable.toDataClass(updatedRow)
+            }
        }
    }

@@ -167,46 +204,4 @@ private class ChapterForDownload(
            },
        )
    }
-
-    private fun updateDownloadStatus(downloaded: Boolean) {
-        transaction {
-            ChapterTable.update({ (ChapterTable.sourceOrder eq chapterIndex) and (ChapterTable.manga eq mangaId) }) {
-                it[isDownloaded] = downloaded
-            }
-        }
-    }
-
-    private fun updateDatabasePages(pageList: List<Page>) {
-        transaction {
-            PageTable.deleteWhere { PageTable.chapter eq chapterId }
-            PageTable.batchInsert(pageList) { page ->
-                this[PageTable.index] = page.index
-                this[PageTable.url] = page.url
-                this[PageTable.imageUrl] = page.imageUrl
-                this[PageTable.chapter] = chapterId
-            }
-        }
-
-        updatePageCount(pageList.size)
-
-        // chapter was updated
-        chapterEntry = freshChapterEntry(chapterId, chapterIndex, mangaId)
-    }
-
-    private fun updatePageCount(pageCount: Int) {
-        transaction {
-            ChapterTable.update({ ChapterTable.id eq chapterId }) {
-                it[ChapterTable.pageCount] = pageCount
-                it[ChapterTable.lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(pageCount - 1).coerceAtLeast(0)
-            }
-        }
-    }
-
-    private fun firstPageExists(): Boolean =
-        try {
-            ChapterDownloadHelper.getImage(mangaId, chapterId, 0).first.close()
-            true
-        } catch (e: Exception) {
-            false
-        }
 }
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/util/DirName.kt
@@ -7,7 +7,6 @@ package suwayomi.tachidesk.manga.impl.util
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at https://mozilla.org/MPL/2.0/. */

-import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.selectAll
 import org.jetbrains.exposed.sql.transactions.transaction
 import suwayomi.tachidesk.manga.impl.util.source.GetCatalogueSource
@@ -20,31 +19,38 @@ import java.io.File

 private val applicationDirs: ApplicationDirs by injectLazy()

-private fun getMangaDir(mangaId: Int): String {
-    val mangaEntry = getMangaEntry(mangaId)
-    val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
+private fun getMangaDir(mangaId: Int): String =
+    transaction {
+        val mangaEntry = MangaTable.selectAll().where { MangaTable.id eq mangaId }.first()
+        val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])

-    val sourceDir = SafePath.buildValidFilename(source.toString())
-    val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title])
-    return "$sourceDir/$mangaDir"
-}
+        val sourceDir = SafePath.buildValidFilename(source.toString())
+        val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title])
+        "$sourceDir/$mangaDir"
+    }

 private fun getChapterDir(
    mangaId: Int,
    chapterId: Int,
-): String {
-    val chapterEntry = transaction { ChapterTable.selectAll().where { ChapterTable.id eq chapterId }.first() }
+): String =
+    transaction {
+        // Get chapter data and build chapter-specific directory name
+        val chapterEntry = ChapterTable.selectAll().where { ChapterTable.id eq chapterId }.first()

-    val chapterDir =
-        SafePath.buildValidFilename(
-            when {
-                chapterEntry[ChapterTable.scanlator] != null -> "${chapterEntry[ChapterTable.scanlator]}_${chapterEntry[ChapterTable.name]}"
-                else -> chapterEntry[ChapterTable.name]
-            },
-        )
+        val chapterDir =
+            SafePath.buildValidFilename(
+                when {
+                    chapterEntry[ChapterTable.scanlator] != null -> {
+                        "${chapterEntry[ChapterTable.scanlator]}_${chapterEntry[ChapterTable.name]}"
+                    }
+                    else -> chapterEntry[ChapterTable.name]
+                },
+            )

-    return getMangaDir(mangaId) + "/$chapterDir"
-}
+        // Get manga directory and combine with chapter directory
+        // Note: This creates a nested transaction, but Exposed handles this with useNestedTransactions=true
+        getMangaDir(mangaId) + "/$chapterDir"
+    }

 fun getThumbnailDownloadPath(mangaId: Int): String = applicationDirs.thumbnailDownloadsRoot + "/$mangaId"

@@ -70,16 +76,21 @@ fun updateMangaDownloadDir(
    mangaId: Int,
    newTitle: String,
 ): Boolean {
-    val mangaEntry = getMangaEntry(mangaId)
-    val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
+    // Get current manga directory (uses its own transaction)
+    val currentMangaDir = getMangaDir(mangaId)

-    val sourceDir = SafePath.buildValidFilename(source.toString())
-    val mangaDir = SafePath.buildValidFilename(mangaEntry[MangaTable.title])
+    // Build new directory path
+    val newMangaDir =
+        transaction {
+            val mangaEntry = MangaTable.selectAll().where { MangaTable.id eq mangaId }.first()
+            val source = GetCatalogueSource.getCatalogueSourceOrStub(mangaEntry[MangaTable.sourceReference])
+            val sourceDir = SafePath.buildValidFilename(source.toString())
+            val newMangaDirName = SafePath.buildValidFilename(newTitle)
+            "$sourceDir/$newMangaDirName"
+        }

-    val newMangaDir = SafePath.buildValidFilename(newTitle)
-
-    val oldDir = "${applicationDirs.downloadsRoot}/$sourceDir/$mangaDir"
-    val newDir = "${applicationDirs.downloadsRoot}/$sourceDir/$newMangaDir"
+    val oldDir = "${applicationDirs.downloadsRoot}/$currentMangaDir"
+    val newDir = "${applicationDirs.downloadsRoot}/$newMangaDir"

    val oldDirFile = File(oldDir)
    val newDirFile = File(newDir)
@@ -90,5 +101,3 @@ fun updateMangaDownloadDir(
        true
    }
 }
-
-private fun getMangaEntry(mangaId: Int): ResultRow = transaction { MangaTable.selectAll().where { MangaTable.id eq mangaId }.first() }