Database improvements

- Move entire sync operation into a single transaction
- Stop loading all manga to memory

server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt

@@ -11,8 +11,9 @@ import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.protobuf.ProtoBuf
+import org.jetbrains.exposed.sql.and
 import org.jetbrains.exposed.sql.selectAll
-import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction
 import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.graphql.types.StartSyncResult
 import suwayomi.tachidesk.manga.impl.Category
@@ -122,16 +123,13 @@ object SyncManager {
     }
 
     private suspend fun syncData() {
-        transaction {
+        newSuspendedTransaction {
             MangaTable.update({ MangaTable.isSyncing eq true }) {
                 it[isSyncing] = false
             }
             ChapterTable.update({ ChapterTable.isSyncing eq true }) {
                 it[isSyncing] = false
             }
-        }
-
-        val databaseManga = getAllMangaThatNeedsSync()
 
             val backupFlags =
                 BackupFlags(
@@ -165,7 +163,7 @@ object SyncManager {
             if (remoteBackup == null) {
                 logger.debug { "Skip restore due to network issues" }
                 // should we call showSyncError?
-            return
+                return@newSuspendedTransaction
             }
 
             if (remoteBackup === syncData.backup) {
@@ -175,22 +173,28 @@ object SyncManager {
                     .edit()
                     .putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
                     .apply()
-            return
+                return@newSuspendedTransaction
             }
 
             // Stop the sync early if the remote backup is null or empty
             if (remoteBackup.backupManga.isEmpty()) {
-            return
+                return@newSuspendedTransaction
             }
 
+            val isLibraryEmpty =
+                MangaTable
+                    .selectAll()
+                    .where { MangaTable.inLibrary eq true }
+                    .empty()
+
             // Check if it's first sync based on lastSyncTimestamp
-        if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && databaseManga.isNotEmpty()) {
+            if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && !isLibraryEmpty) {
                 // It's first sync no need to restore data. (just update remote data)
                 syncPreferences
                     .edit()
                     .putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
                     .apply()
-            return
+                return@newSuspendedTransaction
             }
 
             val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup)
@@ -210,7 +214,7 @@ object SyncManager {
                     .edit()
                     .putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
                     .apply()
-            return
+                return@newSuspendedTransaction
             }
 
             val backupStream = ProtoBuf.encodeToByteArray(Backup.serializer(), newSyncData).inputStream()
@@ -235,12 +239,6 @@ object SyncManager {
                 .putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
                 .apply()
         }
-
-    private fun getAllMangaFromDB(): List<MangaDataClass> = transaction { MangaTable.selectAll().map { MangaTable.toDataClass(it) } }
-
-    private fun getAllMangaThatNeedsSync(): List<MangaDataClass> =
-        transaction {
-            MangaTable.selectAll().where { MangaTable.inLibrary eq true }.map { MangaTable.toDataClass(it) }
     }
 
     private fun isMangaDifferent(
@@ -248,20 +246,17 @@ object SyncManager {
         remoteManga: BackupManga,
     ): Boolean {
         val localChapters =
-            transaction {
             ChapterTable
                 .selectAll()
                 .where { ChapterTable.manga eq localManga.id }
                 .map { ChapterTable.toDataClass(it) }
-            }
+
         val localCategories =
-            transaction {
             CategoryMangaTable
                 .innerJoin(CategoryTable)
                 .selectAll()
                 .where { CategoryMangaTable.manga eq localManga.id }
                 .map { it[CategoryTable.order] }
-            }
 
         if (areChaptersDifferent(localChapters, remoteManga.chapters)) {
             return true
@@ -307,17 +302,20 @@ object SyncManager {
 
         val elapsedTime =
             measureTime {
-                val databaseManga = getAllMangaFromDB()
-                val localMangaMap =
-                    databaseManga.associateBy {
-                        Triple(it.sourceId.toLong(), it.url, it.title)
-                    }
-
                 logger.debug { "Starting to filter favorites and non-favorites from backup data." }
 
                 backup.backupManga.forEach { remoteManga ->
-                    val compositeKey = Triple(remoteManga.source, remoteManga.url, remoteManga.title)
+                    val localManga =
-                    val localManga = localMangaMap[compositeKey]
+                        MangaTable
+                            .selectAll()
+                            .where {
+                                (MangaTable.sourceReference eq remoteManga.source) and
+                                    (MangaTable.url eq remoteManga.url) and
+                                    (MangaTable.title eq remoteManga.title)
+                            }.limit(1)
+                            .map { MangaTable.toDataClass(it) }
+                            .firstOrNull()
+
                     when {
                         // Checks if the manga is in favorites and needs updating or adding
                         remoteManga.favorite -> {
@@ -346,13 +344,19 @@ object SyncManager {
     }
 
     private fun updateNonFavorites(nonFavorites: List<BackupManga>) {
-        val localMangaList = getAllMangaFromDB()
-
-        val localMangaMap = localMangaList.associateBy { Triple(it.sourceId.toLong(), it.url, it.title) }
-
         nonFavorites.forEach { nonFavorite ->
-            val key = Triple(nonFavorite.source, nonFavorite.url, nonFavorite.title)
+            val localManga =
-            localMangaMap[key]?.let { localManga ->
+                MangaTable
+                    .selectAll()
+                    .where {
+                        (MangaTable.sourceReference eq nonFavorite.source) and
+                            (MangaTable.url eq nonFavorite.url) and
+                            (MangaTable.title eq nonFavorite.title)
+                    }.limit(1)
+                    .map { MangaTable.toDataClass(it) }
+                    .firstOrNull()
+
+            if (localManga != null) {
                 if (localManga.inLibrary != nonFavorite.favorite) {
                     val updatedManga = localManga.copy(inLibrary = nonFavorite.favorite)
                     updateManga(updatedManga)
@@ -362,7 +366,6 @@ object SyncManager {
     }
 
     private fun updateManga(manga: MangaDataClass) {
-        transaction {
         MangaTable.update({ MangaTable.id eq manga.id }) {
             it[MangaTable.url] = manga.url
             it[MangaTable.title] = manga.title
@@ -392,4 +395,3 @@ object SyncManager {
         }
     }
 }
-}