#733: Improve perfs on getChapterList with onlineFetch (Less databases calls) (#737)

* improve(#733): less databases calls on getChapterList with onlineFetch

* improve(#733): fixes (delete with ids), tried batch update but not successfull

* improve(#733): fixes (batch update)

* improve(#733): clean imports

* improve(#733): fixes SChapter to ChapterDataClass,

* improve(#733): re-added recognize chap number

---------

Co-authored-by: Alexandre JOURNET <alexandre.journet@axopen.com>

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -18,11 +18,13 @@ import org.jetbrains.exposed.sql.Op
 import org.jetbrains.exposed.sql.ResultRow
 import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.SortOrder.ASC
-import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
+import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
 import org.jetbrains.exposed.sql.and
+import org.jetbrains.exposed.sql.batchInsert
 import org.jetbrains.exposed.sql.deleteWhere
 import org.jetbrains.exposed.sql.insert
 import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.statements.BatchUpdateStatement
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Manga.getManga
@@ -127,46 +129,66 @@ object Chapter {
             chapter.chapter_number = chapterNumber.toFloat()
         }
 
-        var now = Instant.now().epochSecond
+        val now = Instant.now().epochSecond
+        val chaptersInDb =
+            transaction {
+                ChapterTable.select { ChapterTable.manga eq mangaId }
+                    .map { ChapterTable.toDataClass(it) }
+                    .toSet()
+            }
+
+        val chaptersToInsert = mutableListOf<ChapterDataClass>()
+        val chaptersToUpdate = mutableListOf<ChapterDataClass>()
+
+        chapterList.reversed().forEachIndexed { index, fetchedChapter ->
+            val chapterEntry = chaptersInDb.find { it.url == fetchedChapter.url }
+
+            val chapterData =
+                ChapterDataClass.fromSChapter(
+                    fetchedChapter,
+                    chapterEntry?.id ?: 0,
+                    index + 1,
+                    now,
+                    mangaId,
+                    (source as? HttpSource)?.getChapterUrl(fetchedChapter),
+                )
+
+            if (chapterEntry == null) {
+                chaptersToInsert.add(chapterData)
+            } else {
+                chaptersToUpdate.add(chapterData)
+            }
+        }
 
         transaction {
-            chapterList.reversed().forEachIndexed { index, fetchedChapter ->
+            if (chaptersToInsert.isNotEmpty()) {
-                val chapterEntry = ChapterTable.select { ChapterTable.url eq fetchedChapter.url }.firstOrNull()
+                ChapterTable.batchInsert(chaptersToInsert) {
-                if (chapterEntry == null) {
+                    this[ChapterTable.url] = it.url
-                    ChapterTable.insert {
+                    this[ChapterTable.name] = it.name
-                        it[url] = fetchedChapter.url
+                    this[ChapterTable.date_upload] = it.uploadDate
-                        it[name] = fetchedChapter.name
+                    this[ChapterTable.chapter_number] = it.chapterNumber
-                        it[date_upload] = fetchedChapter.date_upload
+                    this[ChapterTable.scanlator] = it.scanlator
-                        it[chapter_number] = fetchedChapter.chapter_number
+                    this[ChapterTable.sourceOrder] = it.index
-                        it[scanlator] = fetchedChapter.scanlator
+                    this[ChapterTable.fetchedAt] = it.fetchedAt
-
+                    this[ChapterTable.manga] = it.mangaId
-                        it[sourceOrder] = index + 1
+                    this[ChapterTable.realUrl] = it.realUrl
-                        it[fetchedAt] = now++
-                        it[ChapterTable.manga] = mangaId
-
-                        it[realUrl] =
-                            runCatching {
-                                (source as? HttpSource)?.getChapterUrl(fetchedChapter)
-                            }.getOrNull()
-                    }
-                } else {
-                    ChapterTable.update({ ChapterTable.url eq fetchedChapter.url }) {
-                        it[name] = fetchedChapter.name
-                        it[date_upload] = fetchedChapter.date_upload
-                        it[chapter_number] = fetchedChapter.chapter_number
-                        it[scanlator] = fetchedChapter.scanlator
-
-                        it[sourceOrder] = index + 1
-                        it[ChapterTable.manga] = mangaId
-
-                        it[realUrl] =
-                            runCatching {
-                                (source as? HttpSource)?.getChapterUrl(fetchedChapter)
-                            }.getOrNull()
-                    }
                 }
             }
 
+            BatchUpdateStatement(ChapterTable).apply {
+                chaptersToUpdate.forEach {
+                    addBatch(EntityID(it.id, ChapterTable))
+                    this[ChapterTable.name] = it.name
+                    this[ChapterTable.date_upload] = it.uploadDate
+                    this[ChapterTable.chapter_number] = it.chapterNumber
+                    this[ChapterTable.scanlator] = it.scanlator
+                    this[ChapterTable.sourceOrder] = it.index
+                    this[ChapterTable.fetchedAt] = it.fetchedAt
+                    this[ChapterTable.realUrl] = it.realUrl
+                }
+                execute(this@transaction)
+            }
+
             MangaTable.update({ MangaTable.id eq mangaId }) {
                 it[MangaTable.chaptersLastFetchedAt] = Instant.now().epochSecond
             }
@@ -186,18 +208,25 @@ object Chapter {
                     ChapterTable.select { ChapterTable.manga eq mangaId }
                         .orderBy(ChapterTable.url to ASC).toList()
                 }
+
             val chapterUrls = chapterList.map { it.url }.toSet()
 
-            dbChapterList.forEachIndexed { index, dbChapter ->
+            val chaptersIdsToDelete =
-                if (
+                dbChapterList.mapIndexedNotNull { index, dbChapter ->
-                    !chapterUrls.contains(dbChapter[ChapterTable.url]) || // is orphaned
+                    val isOrphaned = !chapterUrls.contains(dbChapter[ChapterTable.url])
-                    (index < dbChapterList.lastIndex && dbChapter[ChapterTable.url] == dbChapterList[index + 1][ChapterTable.url]) // is duplicate
+                    val isDuplicate =
-                ) {
+                        index < dbChapterList.lastIndex && dbChapter[ChapterTable.url] == dbChapterList[index + 1][ChapterTable.url]
-                    transaction {
+                    val deleteChapter = isOrphaned || isDuplicate
-                        PageTable.deleteWhere { PageTable.chapter eq dbChapter[ChapterTable.id] }
+                    if (deleteChapter) {
-                        ChapterTable.deleteWhere { ChapterTable.id eq dbChapter[ChapterTable.id] }
+                        dbChapter[ChapterTable.id].value
+                    } else {
+                        null
                     }
                 }
+
+            transaction {
+                PageTable.deleteWhere { PageTable.chapter inList chaptersIdsToDelete }
+                ChapterTable.deleteWhere { ChapterTable.id inList chaptersIdsToDelete }
             }
         }
 
@@ -340,15 +369,19 @@ object Chapter {
                     when {
                         input.chapterIds != null ->
                             Op.build { (ChapterTable.manga eq mangaId) and (ChapterTable.id inList input.chapterIds) }
+
                         input.chapterIndexes != null ->
                             Op.build { (ChapterTable.manga eq mangaId) and (ChapterTable.sourceOrder inList input.chapterIndexes) }
+
                         else -> null
                     }
+
                 else -> {
                     // mangaId is null, only chapterIndexes is valid for this case
                     when {
                         input.chapterIds != null ->
                             Op.build { (ChapterTable.id inList input.chapterIds) }
+
                         else -> null
                     }
                 }

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt

@@ -1,5 +1,7 @@
 package suwayomi.tachidesk.manga.model.dataclass
 
+import eu.kanade.tachiyomi.source.model.SChapter
+
 /*
  * Copyright (C) Contributors to the Suwayomi project
  *
@@ -38,4 +40,33 @@ data class ChapterDataClass(
     val chapterCount: Int? = null,
     /** used to store client specific values */
     val meta: Map<String, String> = emptyMap(),
-)
+) {
+    companion object {
+        fun fromSChapter(
+            sChapter: SChapter,
+            id: Int,
+            index: Int,
+            fetchedAt: Long,
+            mangaId: Int,
+            realUrl: String?,
+        ): ChapterDataClass {
+            return ChapterDataClass(
+                id = id,
+                url = sChapter.url,
+                name = sChapter.name,
+                uploadDate = sChapter.date_upload,
+                chapterNumber = sChapter.chapter_number,
+                scanlator = sChapter.scanlator ?: "",
+                index = index,
+                fetchedAt = fetchedAt,
+                realUrl = realUrl,
+                mangaId = mangaId,
+                read = false,
+                bookmarked = false,
+                lastPageRead = 0,
+                lastReadAt = 0,
+                downloaded = false,
+            )
+        }
+    }
+}