Implement SyncYomi (#1813)

* Implement SyncYomi

* Add ability to select what to sync

* Properly fix default category bug

* Add periodic sync

* Add PostgreSQL support

* Deschedule previous task

* Check if SyncYomi is enabled in syncData function

* Don't allow multiple syncs at the same time

* Convert SyncYomiSyncService to object

* Make startSync non-suspend

* Return a result from startSync

* Sync before library update

* Improvements

* Use NetworkHelper client

* Lint

* Use measureTime

* Database improvements
- Move entire sync operation into a single transaction
- Stop loading all manga to memory

* Revert "Database improvements"

This reverts commit bee8d214c3.

* Actual database improvements

* Remove runBlocking

* Remove title check

* Update updateNonFavorites function

* Update timeout code

* Improve PostgreSQL query

* Create lastSyncState variable

* Create lastSyncStatus query

* Convert lastSyncState to StateFlow

* Create lastSyncStatusChange subscription

* Replace backupRestoreStatus with backupRestoreId

* Add startDate and endDate

* Add logs for sync start and end

* Handle all errors in syncData

* Change category restore function to match Mihon's behavior

* Fix comment

* Remove duplicate BackupMangaHandler.backup call

* Remove duplicated log

* Rename subscription to syncStatusChanged

* Use same flags for restoring

* Update syncInterval config to use DurationSetting

* Update sync scheduling logic

* Reorder conditions to reduce database calls

* Prevent deleted ghost chapters from reappearing during sync
jobobby04/TachiyomiSY#1575

* Improve sync merging categories
jobobby04/TachiyomiSY#1559

* Make columns not null

* Improve H2 triggers

* Add documentation

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt

@@ -85,6 +85,12 @@ object CategoryManga {
         }
     }
 
+    fun removeMangaFromAllCategories(mangaId: Int) {
+        transaction {
+            CategoryMangaTable.deleteWhere { CategoryMangaTable.manga eq mangaId }
+        }
+    }
+
     /**
      * list of mangas that belong to a category
      */

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -129,6 +129,8 @@ object Chapter {
                 downloaded = dbChapter[ChapterTable.isDownloaded],
                 pageCount = dbChapter[ChapterTable.pageCount],
                 chapterCount = chapterList.size,
+                lastModifiedAt = dbChapter[ChapterTable.lastModifiedAt],
+                version = dbChapter[ChapterTable.version],
                 meta = chapterMetas.getValue(dbChapter[ChapterTable.id].value),
             )
         }
@@ -279,6 +281,8 @@ object Chapter {
                                 this[ChapterTable.isRead] = false
                                 this[ChapterTable.isBookmarked] = false
                                 this[ChapterTable.isDownloaded] = false
+                                this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt
+                                this[ChapterTable.version] = chapter.version
                                 this[ChapterTable.pageCount] = -1
 
                                 // is recognized chapter number
@@ -322,6 +326,8 @@ object Chapter {
                                     this[ChapterTable.scanlator] = it.scanlator
                                     this[ChapterTable.sourceOrder] = it.index
                                     this[ChapterTable.realUrl] = it.realUrl
+                                    this[ChapterTable.lastModifiedAt] = it.lastModifiedAt
+                                    this[ChapterTable.version] = it.version
                                     this[ChapterTable.isDownloaded] = currentChapter.downloaded
                                     this[ChapterTable.pageCount] = currentChapter.pageCount
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -99,6 +99,8 @@ object Manga {
                 updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
                 freshData = true,
                 trackers = Track.getTrackRecordsByMangaId(mangaId),
+                lastModifiedAt = mangaEntry[MangaTable.lastModifiedAt],
+                version = mangaEntry[MangaTable.version],
             )
         }
     }
@@ -246,6 +248,8 @@ object Manga {
         updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
         freshData = false,
         trackers = Track.getTrackRecordsByMangaId(mangaId),
+        lastModifiedAt = mangaEntry[MangaTable.lastModifiedAt],
+        version = mangaEntry[MangaTable.version],
     )
 
     fun getMangaMetaMap(mangaId: Int): Map<String, String> =

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt

@@ -21,6 +21,9 @@ import kotlinx.coroutines.sync.withLock
 import okio.buffer
 import okio.gzip
 import okio.source
+import org.jetbrains.exposed.v1.core.eq
+import org.jetbrains.exposed.v1.jdbc.transactions.transaction
+import org.jetbrains.exposed.v1.jdbc.update
 import suwayomi.tachidesk.graphql.types.toStatus
 import suwayomi.tachidesk.manga.impl.backup.BackupFlags
 import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
@@ -31,6 +34,8 @@ import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.MangaTable
 import java.io.InputStream
 import java.util.Date
 import java.util.Timer
@@ -109,6 +114,7 @@ object ProtoBackupImport : ProtoBackupBase() {
     fun restore(
         sourceStream: InputStream,
         flags: BackupFlags,
+        isSync: Boolean = false,
     ): String {
         val restoreId = System.currentTimeMillis().toString()
 
@@ -117,7 +123,7 @@ object ProtoBackupImport : ProtoBackupBase() {
         updateRestoreState(restoreId, BackupRestoreState.Idle)
 
         GlobalScope.launch {
-            restoreLegacy(sourceStream, restoreId, flags)
+            restoreLegacy(sourceStream, restoreId, flags, isSync)
         }
 
         return restoreId
@@ -127,11 +133,12 @@ object ProtoBackupImport : ProtoBackupBase() {
         sourceStream: InputStream,
         restoreId: String = "legacy",
         flags: BackupFlags = BackupFlags.DEFAULT,
+        isSync: Boolean = false,
     ): ValidationResult =
         backupMutex.withLock {
             try {
                 logger.info { "restore($restoreId): restoring..." }
-                performRestore(restoreId, sourceStream, flags)
+                performRestore(restoreId, sourceStream, flags, isSync)
             } catch (e: Exception) {
                 logger.error(e) { "restore($restoreId): failed due to" }
 
@@ -152,12 +159,14 @@ object ProtoBackupImport : ProtoBackupBase() {
         id: String,
         sourceStream: InputStream,
         flags: BackupFlags,
+        isSync: Boolean,
     ): ValidationResult {
         val backupString =
             sourceStream
                 .source()
-                .gzip()
-                .buffer()
+                .run {
+                    if (!isSync) gzip() else this
+                }.buffer()
                 .use { it.readByteArray() }
         val backup = parser.decodeFromByteArray(Backup.serializer(), backupString)
 
@@ -235,6 +244,17 @@ object ProtoBackupImport : ProtoBackupBase() {
             """.trimIndent()
         }
 
+        if (isSync) {
+            transaction {
+                MangaTable.update({ MangaTable.isSyncing eq true }) {
+                    it[isSyncing] = false
+                }
+                ChapterTable.update({ ChapterTable.isSyncing eq true }) {
+                    it[isSyncing] = false
+                }
+            }
+        }
+
         updateRestoreState(id, BackupRestoreState.Success)
 
         return validationResult

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt

@@ -8,7 +8,11 @@ package suwayomi.tachidesk.manga.impl.backup.proto.handlers
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import org.jetbrains.exposed.v1.core.SortOrder
+import org.jetbrains.exposed.v1.core.eq
+import org.jetbrains.exposed.v1.jdbc.insertAndGetId
 import org.jetbrains.exposed.v1.jdbc.selectAll
+import org.jetbrains.exposed.v1.jdbc.transactions.transaction
+import org.jetbrains.exposed.v1.jdbc.update
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas
 import suwayomi.tachidesk.manga.impl.backup.BackupFlags
@@ -38,6 +42,9 @@ object BackupCategoryHandler {
                     it.name,
                     it.order,
                     0, // not supported in Tachidesk
+                    it.version,
+                    it.uid,
+                    it.lastModifiedAt,
                 ).apply {
                     this.meta = categoryToMeta[it.id] ?: emptyMap()
                 }
@@ -45,7 +52,56 @@ object BackupCategoryHandler {
         }
 
     fun restore(backupCategories: List<BackupCategory>): Map<Int, Int> {
-        val categoryIds = Category.createCategories(backupCategories.map { it.name })
+        val dbCategories = Category.getCategoryList()
+        val dbCategoriesByName = dbCategories.associateBy { it.name }
+        val dbCategoriesByUid = dbCategories.associateBy { it.uid }
+
+        var nextOrder = dbCategories.maxOfOrNull { it.order }?.plus(1) ?: 0
+
+        val categoryIds =
+            transaction {
+                backupCategories
+                    .map { backupCategory ->
+                        var dbCategory =
+                            if (backupCategory.uid != 0L) {
+                                dbCategoriesByUid[backupCategory.uid]
+                            } else {
+                                null
+                            }
+
+                        if (dbCategory == null) {
+                            dbCategory = dbCategoriesByName[backupCategory.name]
+                        }
+
+                        if (dbCategory != null) {
+                            CategoryTable.update({ CategoryTable.id eq dbCategory.id }) {
+                                it[name] = backupCategory.name
+                                it[order] = backupCategory.order
+                                it[version] = backupCategory.version
+                                it[uid] = if (backupCategory.uid != 0L) backupCategory.uid else dbCategory.uid
+                                it[lastModifiedAt] = backupCategory.lastModifiedAt
+                                it[isSyncing] = true
+                            }
+                            return@map dbCategory.id
+                        }
+
+                        val currentOrder = nextOrder++
+                        CategoryTable
+                            .insertAndGetId {
+                                it[name] = backupCategory.name
+                                it[order] = currentOrder
+                                it[version] = backupCategory.version
+                                it[uid] = backupCategory.uid
+                                it[lastModifiedAt] = backupCategory.lastModifiedAt
+                            }.value
+                    }
+            }
+
+        transaction {
+            CategoryTable.update({ CategoryTable.isSyncing eq true }) {
+                it[isSyncing] = false
+            }
+        }
 
         val metaEntryByCategoryId =
             categoryIds

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt

@@ -75,6 +75,8 @@ object BackupMangaHandler {
                         dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds,
                         viewer = 0, // not supported in Tachidesk
                         updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]),
+                        lastModifiedAt = mangaRow[MangaTable.lastModifiedAt],
+                        version = mangaRow[MangaTable.version],
                     )
 
                 val mangaId = mangaRow[MangaTable.id].value
@@ -110,6 +112,8 @@ object BackupMangaHandler {
                                     it.uploadDate,
                                     it.chapterNumber,
                                     chapters.size - it.index,
+                                    it.lastModifiedAt,
+                                    it.version,
                                 ).apply {
                                     if (flags.includeClientData) {
                                         this.meta = chapterToMeta[it.id] ?: emptyMap()
@@ -232,6 +236,9 @@ object BackupMangaHandler {
                                 it[inLibrary] = manga.favorite
 
                                 it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
+
+                                it[lastModifiedAt] = manga.lastModifiedAt
+                                it[version] = manga.version
                             }.value
                     } else {
                         val dbMangaId = dbManga[MangaTable.id].value
@@ -251,6 +258,9 @@ object BackupMangaHandler {
                             it[inLibrary] = manga.favorite || dbManga[inLibrary]
 
                             it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
+
+                            it[lastModifiedAt] = manga.lastModifiedAt
+                            it[version] = manga.version
                         }
 
                         dbMangaId
@@ -268,7 +278,7 @@ object BackupMangaHandler {
                     restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags)
                 }
 
-                // merge categories
+                // update categories
                 if (flags.includeCategories) {
                     restoreMangaCategoryData(mangaId, categoryIds)
                 }
@@ -339,6 +349,9 @@ object BackupMangaHandler {
                             this[ChapterTable.lastReadAt] =
                                 historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
                         }
+
+                        this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt
+                        this[ChapterTable.version] = chapter.version
                     }.map { it[ChapterTable.id].value }
             } else {
                 emptyList()
@@ -387,6 +400,7 @@ object BackupMangaHandler {
         mangaId: Int,
         categoryIds: List<Int>,
     ) {
+        CategoryManga.removeMangaFromAllCategories(mangaId)
         CategoryManga.addMangaToCategories(mangaId, categoryIds)
     }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupCategory.kt

@@ -10,6 +10,10 @@ class BackupCategory(
     // @ProtoNumber(3) val updateInterval: Int = 0, 1.x value not used in 0.x
     // Bump by 100 to specify this is a 0.x value
     @ProtoNumber(100) var flags: Int = 0,
+    // syncyomi
+    @ProtoNumber(601) var version: Long = 0,
+    @ProtoNumber(602) var uid: Long = 0,
+    @ProtoNumber(603) var lastModifiedAt: Long = 0,
     // suwayomi
     @ProtoNumber(9000) var meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt

@@ -19,6 +19,9 @@ data class BackupChapter(
     // chapterNumber is called number is 1.x
     @ProtoNumber(9) var chapterNumber: Float = 0F,
     @ProtoNumber(10) var sourceOrder: Int = 0,
+    // syncyomi
+    @ProtoNumber(11) var lastModifiedAt: Long = 0,
+    @ProtoNumber(12) var version: Long = 0,
     // suwayomi
     @ProtoNumber(9000) var meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt

@@ -34,6 +34,9 @@ data class BackupManga(
     @ProtoNumber(103) var viewer_flags: Int? = null,
     @ProtoNumber(104) var history: List<BackupHistory> = emptyList(),
     @ProtoNumber(105) var updateStrategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE,
+    // syncyomi
+    @ProtoNumber(106) var lastModifiedAt: Long = 0,
+    @ProtoNumber(109) var version: Long = 0,
     // suwayomi
     @ProtoNumber(9000) var meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

@@ -28,6 +28,7 @@ import kotlinx.coroutines.flow.update
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Semaphore
 import kotlinx.coroutines.sync.withPermit
+import suwayomi.tachidesk.global.impl.sync.SyncManager
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.Chapter
@@ -337,90 +338,98 @@ class Updater : IUpdater {
         clear: Boolean?,
         forceAll: Boolean,
     ) {
-        saveLastUpdateTimestamp()
-
-        if (clear == true) {
-            reset()
-        }
-
-        val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
-        val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.EXCLUDE].orEmpty()
-        val includedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.INCLUDE].orEmpty()
-        val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.UNSET].orEmpty()
-        val categoriesToUpdate =
-            if (forceAll) {
-                categories
-            } else {
-                includedCategories.ifEmpty { unsetCategories }
-            }
-        val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
-        val updateStatusCategories =
-            mapOf(
-                Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
-                Pair(CategoryUpdateStatus.SKIPPED, skippedCategories),
-            )
-
-        logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
-
-        val categoriesToUpdateMangas =
-            categoriesToUpdate
-                .flatMap { CategoryManga.getCategoryMangaList(it.id) }
-                .distinctBy { it.id }
-        val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
-        val mangasToUpdate =
-            categoriesToUpdateMangas
-                .asSequence()
-                .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
-                .filter {
-                    if (serverConfig.excludeUnreadChapters.value) {
-                        (it.unreadCount ?: 0L) == 0L
-                    } else {
-                        true
-                    }
-                }.filter {
-                    if (it.initialized && serverConfig.excludeNotStarted.value) {
-                        it.lastReadAt != null
-                    } else {
-                        true
-                    }
-                }.filter {
-                    if (serverConfig.excludeCompleted.value) {
-                        it.status != MangaStatus.COMPLETED.name
-                    } else {
-                        true
-                    }
-                }.filter { forceAll || !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
-                .toList()
-        val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
-
-        this.updateStatusCategories = updateStatusCategories
-        this.updateStatusSkippedMangas = skippedMangas
-
-        if (mangasToUpdate.isEmpty()) {
-            // In case no manga gets updated and no update job was running before, the client would never receive an info
-            // about its update request
-            scope.launch {
-                updateStatus(immediate = true)
-            }
-            return
-        }
-
         scope.launch {
-            updateStatus(
-                categoryUpdates =
-                    updateStatusCategories[CategoryUpdateStatus.UPDATING]
-                        ?.map {
-                            CategoryUpdateJob(it, CategoryUpdateStatus.UPDATING)
-                        }.orEmpty(),
-                mangaUpdates = mangasToUpdate.map { UpdateJob(it) },
-                isRunning = true,
+            SyncManager.ensureSync()
+
+            saveLastUpdateTimestamp()
+
+            if (clear == true) {
+                reset()
+            }
+
+            val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
+            val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.EXCLUDE].orEmpty()
+            val includedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.INCLUDE].orEmpty()
+            val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.UNSET].orEmpty()
+            val categoriesToUpdate =
+                if (forceAll) {
+                    categories
+                } else {
+                    includedCategories.ifEmpty { unsetCategories }
+                }
+            val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
+            val updateStatusCategories =
+                mapOf(
+                    Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
+                    Pair(CategoryUpdateStatus.SKIPPED, skippedCategories),
+                )
+
+            logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
+
+            val categoriesToUpdateMangas =
+                categoriesToUpdate
+                    .flatMap { CategoryManga.getCategoryMangaList(it.id) }
+                    .distinctBy { it.id }
+            val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
+            val mangasToUpdate =
+                categoriesToUpdateMangas
+                    .asSequence()
+                    .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
+                    .filter {
+                        if (serverConfig.excludeUnreadChapters.value) {
+                            (it.unreadCount ?: 0L) == 0L
+                        } else {
+                            true
+                        }
+                    }.filter {
+                        if (it.initialized && serverConfig.excludeNotStarted.value) {
+                            it.lastReadAt != null
+                        } else {
+                            true
+                        }
+                    }.filter {
+                        if (serverConfig.excludeCompleted.value) {
+                            it.status != MangaStatus.COMPLETED.name
+                        } else {
+                            true
+                        }
+                    }.filter {
+                        forceAll ||
+                            !excludedCategories.any { category ->
+                                mangasToCategoriesMap[it.id]?.contains(category) == true
+                            }
+                    }.toList()
+            val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
+
+            this@Updater.updateStatusCategories = updateStatusCategories
+            this@Updater.updateStatusSkippedMangas = skippedMangas
+
+            if (mangasToUpdate.isEmpty()) {
+                // In case no manga gets updated and no update job was running before, the client would never receive an info
+                // about its update request
+                scope.launch {
+                    updateStatus(immediate = true)
+                }
+                return@launch
+            }
+
+            scope.launch {
+                updateStatus(
+                    categoryUpdates =
+                        updateStatusCategories[CategoryUpdateStatus.UPDATING]
+                            ?.map {
+                                CategoryUpdateJob(it, CategoryUpdateStatus.UPDATING)
+                            }.orEmpty(),
+                    mangaUpdates = mangasToUpdate.map { UpdateJob(it) },
+                    isRunning = true,
+                )
+            }
+
+            addMangasToQueue(
+                mangasToUpdate
+                    .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)),
             )
         }
-
-        addMangasToQueue(
-            mangasToUpdate
-                .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)),
-        )
     }
 
     override fun addMangasToQueue(mangas: List<MangaDataClass>) {