Implement SyncYomi

server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt

@@ -0,0 +1,298 @@
+package suwayomi.tachidesk.global.impl.sync
+
+import android.app.Application
+import android.content.Context
+import io.github.oshai.kotlinlogging.KotlinLogging
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoBuf
+import org.jetbrains.exposed.sql.selectAll
+import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
+import suwayomi.tachidesk.manga.impl.backup.BackupFlags
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport
+import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler
+import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
+import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
+import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
+import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
+import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
+import suwayomi.tachidesk.manga.model.table.CategoryMangaTable
+import suwayomi.tachidesk.manga.model.table.CategoryTable
+import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.MangaStatus
+import suwayomi.tachidesk.manga.model.table.MangaTable
+import suwayomi.tachidesk.manga.model.table.toDataClass
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import java.util.Date
+import kotlin.system.measureTimeMillis
+
+@Serializable
+data class SyncData(
+    val backup: Backup? = null,
+)
+
+object SyncManager {
+    private val syncPreferences = Injekt.get<Application>().getSharedPreferences("sync", Context.MODE_PRIVATE)
+    private val logger = KotlinLogging.logger {}
+
+    suspend fun syncData() {
+        transaction {
+            MangaTable.update({ MangaTable.isSyncing eq true }) {
+                it[isSyncing] = false
+            }
+            ChapterTable.update({ ChapterTable.isSyncing eq true }) {
+                it[isSyncing] = false
+            }
+        }
+
+        val databaseManga = getAllMangaThatNeedsSync()
+
+        val backupFlags = BackupFlags(
+            includeManga = true,
+            includeCategories = true,
+            includeChapters = true,
+            includeTracking = true,
+            includeHistory = true,
+            includeClientData = false,
+            includeServerSettings = false,
+        )
+
+        val backupMangas = BackupMangaHandler.backup(backupFlags)
+
+        val backup = Backup(
+            BackupMangaHandler.backup(backupFlags),
+            BackupCategoryHandler.backup(backupFlags),
+            BackupSourceHandler.backup(backupMangas, backupFlags),
+            emptyMap(),
+            null,
+        )
+
+        val syncData = SyncData(
+            backup = backup,
+        )
+
+        val remoteBackup = SyncYomiSyncService().doSync(syncData)
+
+        if (remoteBackup == null) {
+            logger.debug { "Skip restore due to network issues" }
+            // should we call showSyncError?
+            return
+        }
+
+        if (remoteBackup === syncData.backup) {
+            // nothing changed
+            logger.debug { "Skip restore due to remote was overwrite from local" }
+            syncPreferences.edit()
+                .putLong("last_sync_timestamp", Date().time)
+                .apply()
+            return
+        }
+
+        // Stop the sync early if the remote backup is null or empty
+        if (remoteBackup.backupManga.isEmpty()) {
+            return
+        }
+
+        // Check if it's first sync based on lastSyncTimestamp
+        if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && databaseManga.isNotEmpty()) {
+            // It's first sync no need to restore data. (just update remote data)
+            syncPreferences.edit()
+                .putLong("last_sync_timestamp", Date().time)
+                .apply()
+            return
+        }
+
+        val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup)
+        updateNonFavorites(nonFavorites)
+
+        val newSyncData = backup.copy(
+            backupManga = filteredFavorites,
+            backupCategories = remoteBackup.backupCategories,
+            backupSources = remoteBackup.backupSources,
+        )
+
+        // It's local sync no need to restore data. (just update remote data)
+        if (filteredFavorites.isEmpty()) {
+            // update the sync timestamp
+            syncPreferences.edit()
+                .putLong("last_sync_timestamp", Date().time)
+                .apply()
+            return
+        }
+
+        val backupStream = ProtoBuf.encodeToByteArray(Backup.serializer(), newSyncData).inputStream()
+        ProtoBackupImport.restore(
+            sourceStream = backupStream,
+            flags = BackupFlags(
+                includeManga = true,
+                includeCategories = true,
+                includeChapters = true,
+                includeTracking = true,
+                includeHistory = true,
+                includeClientData = false,
+                includeServerSettings = false,
+            ),
+            isSync = true,
+        )
+
+        // update the sync timestamp
+        syncPreferences.edit()
+            .putLong("last_sync_timestamp", Date().time)
+            .apply()
+    }
+
+    private fun getAllMangaFromDB(): List<MangaDataClass> {
+        return transaction { MangaTable.selectAll().map { MangaTable.toDataClass(it) } }
+    }
+
+    private fun getAllMangaThatNeedsSync(): List<MangaDataClass> {
+        return transaction {
+            MangaTable.selectAll().where { MangaTable.inLibrary eq true }.map { MangaTable.toDataClass(it) }
+        }
+    }
+
+    private fun isMangaDifferent(localManga: MangaDataClass, remoteManga: BackupManga): Boolean {
+        val localChapters = transaction {
+            ChapterTable
+                .selectAll()
+                .where { ChapterTable.manga eq localManga.id }
+                .map { ChapterTable.toDataClass(it) }
+        }
+        val localCategories = transaction {
+            CategoryMangaTable
+                .innerJoin(CategoryTable)
+                .selectAll()
+                .where { CategoryMangaTable.manga eq localManga.id }
+                .map { it[CategoryTable.order] }
+        }
+
+        if (areChaptersDifferent(localChapters, remoteManga.chapters)) {
+            return true
+        }
+
+        if (localManga.version != remoteManga.version) {
+            return true
+        }
+
+        if (localCategories.toSet() != remoteManga.categories.toSet()) {
+            return true
+        }
+
+        return false
+    }
+
+    private fun areChaptersDifferent(
+        localChapters: List<ChapterDataClass>,
+        remoteChapters: List<BackupChapter>,
+    ): Boolean {
+        val localChapterMap = localChapters.associateBy { it.url }
+        val remoteChapterMap = remoteChapters.associateBy { it.url }
+
+        if (localChapterMap.size != remoteChapterMap.size) {
+            return true
+        }
+
+        for ((url, localChapter) in localChapterMap) {
+            val remoteChapter = remoteChapterMap[url]
+
+            // If a matching remote chapter doesn't exist, or the version numbers are different, consider them different
+            if (remoteChapter == null || localChapter.version != remoteChapter.version) {
+                return true
+            }
+        }
+
+        return false
+    }
+
+    private fun filterFavoritesAndNonFavorites(backup: Backup): Pair<List<BackupManga>, List<BackupManga>> {
+        val favorites = mutableListOf<BackupManga>()
+        val nonFavorites = mutableListOf<BackupManga>()
+
+        val elapsedTimeMillis = measureTimeMillis {
+            val databaseManga = getAllMangaFromDB()
+            val localMangaMap = databaseManga.associateBy {
+                Triple(it.sourceId.toLong(), it.url, it.title)
+            }
+
+            logger.debug { "Starting to filter favorites and non-favorites from backup data." }
+
+            backup.backupManga.forEach { remoteManga ->
+                val compositeKey = Triple(remoteManga.source, remoteManga.url, remoteManga.title)
+                val localManga = localMangaMap[compositeKey]
+                when {
+                    // Checks if the manga is in favorites and needs updating or adding
+                    remoteManga.favorite -> {
+                        if (localManga == null || isMangaDifferent(localManga, remoteManga)) {
+                            logger.debug { "Adding to favorites: ${remoteManga.title}" }
+                            favorites.add(remoteManga)
+                        } else {
+                            logger.debug { "Already up-to-date favorite: ${remoteManga.title}" }
+                        }
+                    }
+                    // Handle non-favorites
+                    !remoteManga.favorite -> {
+                        logger.debug { "Adding to non-favorites: ${remoteManga.title}" }
+                        nonFavorites.add(remoteManga)
+                    }
+                }
+            }
+        }
+
+        val minutes = elapsedTimeMillis / 60000
+        val seconds = (elapsedTimeMillis % 60000) / 1000
+        logger.debug { "Filtering completed in ${minutes}m ${seconds}s. Favorites found: ${favorites.size}, Non-favorites found: ${nonFavorites.size}" }
+
+        return Pair(favorites, nonFavorites)
+    }
+
+    private fun updateNonFavorites(nonFavorites: List<BackupManga>) {
+        val localMangaList = getAllMangaFromDB()
+
+        val localMangaMap = localMangaList.associateBy { Triple(it.sourceId.toLong(), it.url, it.title) }
+
+        nonFavorites.forEach { nonFavorite ->
+            val key = Triple(nonFavorite.source, nonFavorite.url, nonFavorite.title)
+            localMangaMap[key]?.let { localManga ->
+                if (localManga.inLibrary != nonFavorite.favorite) {
+                    val updatedManga = localManga.copy(inLibrary = nonFavorite.favorite)
+                    updateManga(updatedManga)
+                }
+            }
+        }
+    }
+
+    private fun updateManga(manga: MangaDataClass) {
+        transaction {
+            MangaTable.update({ MangaTable.id eq manga.id }) {
+                it[MangaTable.url] = manga.url
+                it[MangaTable.title] = manga.title
+                it[MangaTable.initialized] = manga.initialized
+
+                it[MangaTable.artist] = manga.artist
+                it[MangaTable.author] = manga.author
+                it[MangaTable.description] = manga.description
+                it[MangaTable.genre] = manga.genre.joinToString(separator = ", ")
+
+                it[MangaTable.status] = MangaStatus.valueOf(manga.status).value
+                it[MangaTable.thumbnail_url] = manga.thumbnailUrl
+                it[MangaTable.thumbnailUrlLastFetched] = manga.thumbnailUrlLastFetched
+
+                it[MangaTable.inLibrary] = manga.inLibrary
+                it[MangaTable.inLibraryAt] = manga.inLibraryAt
+
+                it[MangaTable.sourceReference] = manga.sourceId.toLongOrNull() ?: 0L
+
+                it[MangaTable.realUrl] = manga.realUrl
+                it[MangaTable.lastFetchedAt] = manga.lastFetchedAt ?: 0L
+                it[MangaTable.chaptersLastFetchedAt] = manga.chaptersLastFetchedAt ?: 0L
+
+                it[MangaTable.updateStrategy] = manga.updateStrategy.name
+
+                it[MangaTable.version] = manga.version
+            }
+        }
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt

@@ -0,0 +1,401 @@
+package suwayomi.tachidesk.global.impl.sync
+
+import android.app.Application
+import android.content.Context
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.PUT
+import eu.kanade.tachiyomi.network.await
+import io.github.oshai.kotlinlogging.KotlinLogging
+import io.javalin.http.HttpStatus
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.protobuf.ProtoBuf
+import okhttp3.Headers
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.RequestBody.Companion.toRequestBody
+import suwayomi.tachidesk.manga.impl.Category
+import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
+import suwayomi.tachidesk.server.serverConfig
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import java.util.concurrent.TimeUnit
+
+class SyncYomiSyncService {
+    private val syncPreferences = Injekt.get<Application>().getSharedPreferences("sync", Context.MODE_PRIVATE)
+
+    private val logger = KotlinLogging.logger {}
+
+    private class SyncYomiException(message: String?) : Exception(message)
+
+    suspend fun doSync(syncData: SyncData): Backup? {
+        try {
+            val (remoteData, etag) = pullSyncData()
+
+            val finalSyncData = if (remoteData != null) {
+                assert(etag.isNotEmpty()) { "ETag should never be empty if remote data is not null" }
+                logger.debug { "Try update remote data with ETag($etag)" }
+                mergeSyncData(syncData, remoteData)
+            } else {
+                // init or overwrite remote data
+                logger.debug { "Try overwrite remote data with ETag($etag)" }
+                syncData
+            }
+
+            pushSyncData(finalSyncData, etag)
+            return finalSyncData.backup
+        } catch (e: Exception) {
+            logger.error { "Error syncing: ${e.message}" }
+            return null
+        }
+    }
+
+    private suspend fun pullSyncData(): Pair<SyncData?, String> {
+        val host = serverConfig.syncYomiHost.value
+        val apiKey = serverConfig.syncYomiApiKey.value
+        val downloadUrl = "$host/api/sync/content"
+
+        val headersBuilder = Headers.Builder().add("X-API-Token", apiKey)
+        val lastETag = syncPreferences.getString("last_sync_etag", "") ?: ""
+        if (lastETag != "") {
+            headersBuilder.add("If-None-Match", lastETag)
+        }
+        val headers = headersBuilder.build()
+
+        val downloadRequest = GET(
+            url = downloadUrl,
+            headers = headers,
+        )
+
+        val client = OkHttpClient()
+        val response = client.newCall(downloadRequest).await()
+
+        if (response.code == HttpStatus.NOT_MODIFIED.code) {
+            // not modified
+            assert(lastETag.isNotEmpty())
+            logger.info { "Remote server not modified" }
+            return Pair(null, lastETag)
+        } else if (response.code == HttpStatus.NOT_FOUND.code) {
+            // maybe got deleted from remote
+            return Pair(null, "")
+        }
+
+        if (response.isSuccessful) {
+            val newETag = response.headers["ETag"]
+                .takeIf { it?.isNotEmpty() == true } ?: throw SyncYomiException("Missing ETag")
+
+            val byteArray = response.body.byteStream().use {
+                return@use it.readBytes()
+            }
+
+            return try {
+                val backup = ProtoBuf.decodeFromByteArray(Backup.serializer(), byteArray)
+                return Pair(SyncData(backup = backup), newETag)
+            } catch (_: SerializationException) {
+                logger.info { "Bad content responsed from server" }
+                // the body is invalid
+                // return default value so we can overwrite it
+                Pair(null, "")
+            }
+        } else {
+            val responseBody = response.body.string()
+            logger.error { "SyncError: $responseBody" }
+            throw SyncYomiException("Failed to download sync data: $responseBody")
+        }
+    }
+
+    private suspend fun pushSyncData(syncData: SyncData, eTag: String) {
+        val backup = syncData.backup ?: return
+
+        val host = serverConfig.syncYomiHost.value
+        val apiKey = serverConfig.syncYomiApiKey.value
+        val uploadUrl = "$host/api/sync/content"
+        val timeout = 30L
+
+        val headersBuilder = Headers.Builder().add("X-API-Token", apiKey)
+        if (eTag.isNotEmpty()) {
+            headersBuilder.add("If-Match", eTag)
+        }
+        val headers = headersBuilder.build()
+
+        // Set timeout to 30 seconds
+        val client = OkHttpClient.Builder()
+            .connectTimeout(timeout, TimeUnit.SECONDS)
+            .readTimeout(timeout, TimeUnit.SECONDS)
+            .writeTimeout(timeout, TimeUnit.SECONDS)
+            .build()
+
+        val byteArray = ProtoBuf.encodeToByteArray(Backup.serializer(), backup)
+        if (byteArray.isEmpty()) {
+            throw IllegalStateException("Empty backup error")
+        }
+        val body = byteArray.toRequestBody("application/octet-stream".toMediaType())
+
+        val uploadRequest = PUT(
+            url = uploadUrl,
+            headers = headers,
+            body = body,
+        )
+
+        val response = client.newCall(uploadRequest).await()
+
+        if (response.isSuccessful) {
+            val newETag = response.headers["ETag"]
+                .takeIf { it?.isNotEmpty() == true } ?: throw SyncYomiException("Missing ETag")
+            syncPreferences.edit()
+                .putString("last_sync_etag", newETag)
+                .apply()
+            logger.debug { "SyncYomi sync completed" }
+        } else if (response.code == HttpStatus.PRECONDITION_FAILED.code) {
+            // other clients updated remote data, will try next time
+            logger.debug { "SyncYomi sync failed with 412" }
+        } else {
+            val responseBody = response.body.string()
+            logger.error { "SyncError: $responseBody" }
+        }
+    }
+
+    fun mergeSyncData(localSyncData: SyncData, remoteSyncData: SyncData): SyncData {
+        val mergedCategoriesList =
+            mergeCategoriesLists(localSyncData.backup?.backupCategories, remoteSyncData.backup?.backupCategories)
+
+        val mergedMangaList = mergeMangaLists(
+            localSyncData.backup?.backupManga,
+            remoteSyncData.backup?.backupManga,
+            localSyncData.backup?.backupCategories ?: emptyList(),
+            remoteSyncData.backup?.backupCategories ?: emptyList(),
+            mergedCategoriesList,
+        )
+
+        val mergedSourcesList =
+            mergeSourcesLists(localSyncData.backup?.backupSources, remoteSyncData.backup?.backupSources)
+
+        // Create the merged Backup object
+        val mergedBackup = Backup(
+            backupManga = mergedMangaList,
+            backupCategories = mergedCategoriesList,
+            backupSources = mergedSourcesList,
+            meta = emptyMap(),
+            serverSettings = null,
+        )
+
+        // Create the merged SData object
+        return SyncData(
+            backup = mergedBackup,
+        )
+    }
+
+    private fun mergeMangaLists(
+        localMangaList: List<BackupManga>?,
+        remoteMangaList: List<BackupManga>?,
+        localCategories: List<BackupCategory>,
+        remoteCategories: List<BackupCategory>,
+        mergedCategories: List<BackupCategory>,
+    ): List<BackupManga> {
+        val localMangaListSafe = localMangaList.orEmpty()
+        val remoteMangaListSafe = remoteMangaList.orEmpty()
+
+        logger.debug { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" }
+
+        fun mangaCompositeKey(manga: BackupManga): String {
+            return "${manga.source}|${manga.url}|${manga.title.lowercase().trim()}|${manga.author?.lowercase()?.trim()}"
+        }
+
+        // Create maps using composite keys
+        val localMangaMap = localMangaListSafe.associateBy { mangaCompositeKey(it) }
+        val remoteMangaMap = remoteMangaListSafe.associateBy { mangaCompositeKey(it) }
+
+        val localCategoriesMapByOrder = localCategories.associateBy { it.order }
+        val remoteCategoriesMapByOrder = remoteCategories.associateBy { it.order }
+        val mergedCategoriesMapByName = mergedCategories.associateBy { it.name }
+
+        fun updateCategories(theManga: BackupManga, theMap: Map<Int, BackupCategory>): BackupManga {
+            return theManga.copy(
+                categories = theManga.categories.mapNotNull {
+                    theMap[it]?.let { category ->
+                        mergedCategoriesMapByName[category.name]?.order
+                    }
+                },
+            )
+        }
+
+        logger.debug { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" }
+
+        val mergedList = (localMangaMap.keys + remoteMangaMap.keys).distinct().mapNotNull { compositeKey ->
+            val local = localMangaMap[compositeKey]
+            val remote = remoteMangaMap[compositeKey]
+
+            // New version comparison logic
+            when {
+                local != null && remote == null -> updateCategories(local, localCategoriesMapByOrder)
+                local == null && remote != null -> updateCategories(remote, remoteCategoriesMapByOrder)
+                local != null && remote != null -> {
+                    // Compare versions to decide which manga to keep
+                    if (local.version >= remote.version) {
+                        logger.debug { "Keeping local version of ${local.title} with merged chapters." }
+                        updateCategories(
+                            local.copy(chapters = mergeChapters(local.chapters, remote.chapters)),
+                            localCategoriesMapByOrder,
+                        )
+                    } else {
+                        logger.debug { "Keeping remote version of ${remote.title} with merged chapters." }
+                        updateCategories(
+                            remote.copy(chapters = mergeChapters(local.chapters, remote.chapters)),
+                            remoteCategoriesMapByOrder,
+                        )
+                    }
+                }
+
+                else -> null // No manga found for key
+            }
+        }
+
+        // Counting favorites and non-favorites
+        val (favorites, nonFavorites) = mergedList.partition { it.favorite }
+
+        logger.debug { "Merge completed. Total merged manga: ${mergedList.size}, Favorites: ${favorites.size}, Non-Favorites: ${nonFavorites.size}" }
+
+        return mergedList
+    }
+
+    private fun mergeChapters(
+        localChapters: List<BackupChapter>,
+        remoteChapters: List<BackupChapter>,
+    ): List<BackupChapter> {
+        fun chapterCompositeKey(chapter: BackupChapter): String {
+            return "${chapter.url}|${chapter.name}|${chapter.chapterNumber}"
+        }
+
+        val localChapterMap = localChapters.associateBy { chapterCompositeKey(it) }
+        val remoteChapterMap = remoteChapters.associateBy { chapterCompositeKey(it) }
+
+        logger.debug { "Starting chapter merge. Local chapters: ${localChapters.size}, Remote chapters: ${remoteChapters.size}" }
+
+        // Merge both chapter maps based on version numbers
+        val mergedChapters = (localChapterMap.keys + remoteChapterMap.keys).distinct().mapNotNull { compositeKey ->
+            val localChapter = localChapterMap[compositeKey]
+            val remoteChapter = remoteChapterMap[compositeKey]
+
+            logger.debug { "Processing chapter key: $compositeKey. Local chapter: ${localChapter != null}, Remote chapter: ${remoteChapter != null}" }
+
+            when {
+                localChapter != null && remoteChapter == null -> {
+                    logger.debug { "Keeping local chapter: ${localChapter.name}." }
+                    localChapter
+                }
+
+                localChapter == null && remoteChapter != null -> {
+                    logger.debug { "Taking remote chapter: ${remoteChapter.name}." }
+                    remoteChapter
+                }
+
+                localChapter != null && remoteChapter != null -> {
+                    // Use version number to decide which chapter to keep
+                    val chosenChapter = if (localChapter.version >= remoteChapter.version) {
+                        // If there mare more chapter on remote, local sourceOrder will need to be updated to maintain correct source order.
+                        if (localChapters.size < remoteChapters.size) {
+                            localChapter.copy(sourceOrder = remoteChapter.sourceOrder)
+                        } else {
+                            localChapter
+                        }
+                    } else {
+                        remoteChapter
+                    }
+                    logger.debug { "Merging chapter: ${chosenChapter.name}. Chosen version from: ${if (localChapter.version >= remoteChapter.version) "Local" else "Remote"}, Local version: ${localChapter.version}, Remote version: ${remoteChapter.version}." }
+                    chosenChapter
+                }
+
+                else -> {
+                    logger.debug { "No chapter found for composite key: $compositeKey. Skipping." }
+                    null
+                }
+            }
+        }
+
+        logger.debug { "Chapter merge completed. Total merged chapters: ${mergedChapters.size}" }
+
+        return mergedChapters
+    }
+
+    private fun mergeCategoriesLists(
+        localCategoriesList: List<BackupCategory>?,
+        remoteCategoriesList: List<BackupCategory>?,
+    ): List<BackupCategory> {
+        if (localCategoriesList == null) return remoteCategoriesList ?: emptyList()
+        if (remoteCategoriesList == null) return localCategoriesList
+        val localCategoriesMap = localCategoriesList
+            .filter { it.name != Category.DEFAULT_CATEGORY_NAME }
+            .associateBy { it.name }
+        val remoteCategoriesMap = remoteCategoriesList.associateBy { it.name }
+
+        val mergedCategoriesMap = mutableMapOf<String, BackupCategory>()
+
+        localCategoriesMap.forEach { (name, localCategory) ->
+            val remoteCategory = remoteCategoriesMap[name]
+            if (remoteCategory != null) {
+                // Compare and merge local and remote categories
+                val mergedCategory = if (localCategory.order > remoteCategory.order) {
+                    localCategory
+                } else {
+                    remoteCategory
+                }
+                mergedCategoriesMap[name] = mergedCategory
+            } else {
+                // If the category is only in the local list, add it to the merged list
+                mergedCategoriesMap[name] = localCategory
+            }
+        }
+
+        // Add any categories from the remote list that are not in the local list
+        remoteCategoriesMap.forEach { (name, remoteCategory) ->
+            if (!mergedCategoriesMap.containsKey(name)) {
+                mergedCategoriesMap[name] = remoteCategory
+            }
+        }
+
+        return mergedCategoriesMap.values.toList()
+    }
+
+    private fun mergeSourcesLists(
+        localSources: List<BackupSource>?,
+        remoteSources: List<BackupSource>?,
+    ): List<BackupSource> {
+        // Create maps using sourceId as key
+        val localSourceMap = localSources?.associateBy { it.sourceId } ?: emptyMap()
+        val remoteSourceMap = remoteSources?.associateBy { it.sourceId } ?: emptyMap()
+
+        logger.debug { "Starting source merge. Local sources: ${localSources?.size}, Remote sources: ${remoteSources?.size}" }
+
+        // Merge both source maps
+        val mergedSources = (localSourceMap.keys + remoteSourceMap.keys).distinct().mapNotNull { sourceId ->
+            val localSource = localSourceMap[sourceId]
+            val remoteSource = remoteSourceMap[sourceId]
+
+            logger.debug { "Processing source ID: $sourceId. Local source: ${localSource != null}, Remote source: ${remoteSource != null}" }
+
+            when {
+                localSource != null && remoteSource == null -> {
+                    logger.debug { "Using local source: ${localSource.name}." }
+                    localSource
+                }
+
+                remoteSource != null && localSource == null -> {
+                    logger.debug { "Using remote source: ${remoteSource.name}." }
+                    remoteSource
+                }
+
+                else -> {
+                    logger.debug { "Remote and local is not empty: $sourceId. Skipping." }
+                    null
+                }
+            }
+        }
+
+        logger.debug { "Source merge completed. Total merged sources: ${mergedSources.size}" }
+
+        return mergedSources
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt

@@ -0,0 +1,34 @@
+package suwayomi.tachidesk.graphql.mutations
+
+import kotlinx.coroutines.DelicateCoroutinesApi
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.launch
+import suwayomi.tachidesk.global.impl.sync.SyncManager
+import suwayomi.tachidesk.graphql.directives.RequireAuth
+import suwayomi.tachidesk.server.serverConfig
+
+class SyncMutation {
+    data class StartSyncInput(
+        val clientMutationId: String? = null,
+    )
+
+    data class StartSyncPayload(
+        val clientMutationId: String? = null,
+    )
+
+    @OptIn(DelicateCoroutinesApi::class)
+    @RequireAuth
+    fun startSync(input: StartSyncInput): StartSyncPayload {
+        val (clientMutationId) = input
+
+        if (serverConfig.syncYomiEnabled.value) {
+            GlobalScope.launch {
+                SyncManager.syncData()
+            }
+        }
+
+        return StartSyncPayload(
+            clientMutationId = clientMutationId,
+        )
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt

@@ -27,6 +27,7 @@ import suwayomi.tachidesk.graphql.mutations.MangaMutation
 import suwayomi.tachidesk.graphql.mutations.MetaMutation
 import suwayomi.tachidesk.graphql.mutations.SettingsMutation
 import suwayomi.tachidesk.graphql.mutations.SourceMutation
+import suwayomi.tachidesk.graphql.mutations.SyncMutation
 import suwayomi.tachidesk.graphql.mutations.TrackMutation
 import suwayomi.tachidesk.graphql.mutations.UpdateMutation
 import suwayomi.tachidesk.graphql.mutations.UserMutation
@@ -114,6 +115,7 @@ val schema =
                 TopLevelObject(MangaMutation()),
                 TopLevelObject(MetaMutation()),
                 TopLevelObject(SettingsMutation()),
+                TopLevelObject(SyncMutation()),
                 TopLevelObject(SourceMutation()),
                 TopLevelObject(TrackMutation()),
                 TopLevelObject(UpdateMutation()),

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -125,6 +125,7 @@ object Chapter {
                 downloaded = dbChapter[ChapterTable.isDownloaded],
                 pageCount = dbChapter[ChapterTable.pageCount],
                 chapterCount = chapterList.size,
+                version = dbChapter[ChapterTable.version],
                 meta = chapterMetas.getValue(dbChapter[ChapterTable.id].value),
             )
         }
@@ -283,6 +284,7 @@ object Chapter {
                                 this[ChapterTable.isRead] = false
                                 this[ChapterTable.isBookmarked] = false
                                 this[ChapterTable.isDownloaded] = false
+                                this[ChapterTable.version] = chapter.version
 
                                 // is recognized chapter number
                                 if (chapter.chapterNumber >= 0f && chapter.chapterNumber in deletedChapterNumbers) {
@@ -315,6 +317,7 @@ object Chapter {
                                 this[ChapterTable.scanlator] = it.scanlator
                                 this[ChapterTable.sourceOrder] = it.index
                                 this[ChapterTable.realUrl] = it.realUrl
+                                this[ChapterTable.version] = it.version
                             }
                             execute(this@transaction)
                         }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -96,6 +96,7 @@ object Manga {
                 updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
                 freshData = true,
                 trackers = Track.getTrackRecordsByMangaId(mangaId),
+                version = mangaEntry[MangaTable.version],
             )
         }
     }
@@ -243,6 +244,7 @@ object Manga {
         updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
         freshData = false,
         trackers = Track.getTrackRecordsByMangaId(mangaId),
+        version = mangaEntry[MangaTable.version],
     )
 
     fun getMangaMetaMap(mangaId: Int): Map<String, String> =

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt

@@ -21,6 +21,8 @@ import kotlinx.coroutines.sync.withLock
 import okio.buffer
 import okio.gzip
 import okio.source
+import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.graphql.types.toStatus
 import suwayomi.tachidesk.manga.impl.backup.BackupFlags
 import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
@@ -31,6 +33,8 @@ import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.MangaTable
 import java.io.InputStream
 import java.util.Date
 import java.util.Timer
@@ -109,6 +113,7 @@ object ProtoBackupImport : ProtoBackupBase() {
     fun restore(
         sourceStream: InputStream,
         flags: BackupFlags,
+        isSync: Boolean = false,
     ): String {
         val restoreId = System.currentTimeMillis().toString()
 
@@ -117,7 +122,7 @@ object ProtoBackupImport : ProtoBackupBase() {
         updateRestoreState(restoreId, BackupRestoreState.Idle)
 
         GlobalScope.launch {
-            restoreLegacy(sourceStream, restoreId, flags)
+            restoreLegacy(sourceStream, restoreId, flags, isSync)
         }
 
         return restoreId
@@ -127,11 +132,12 @@ object ProtoBackupImport : ProtoBackupBase() {
         sourceStream: InputStream,
         restoreId: String = "legacy",
         flags: BackupFlags = BackupFlags.DEFAULT,
+        isSync: Boolean = false,
     ): ValidationResult =
         backupMutex.withLock {
             try {
                 logger.info { "restore($restoreId): restoring..." }
-                performRestore(restoreId, sourceStream, flags)
+                performRestore(restoreId, sourceStream, flags, isSync)
             } catch (e: Exception) {
                 logger.error(e) { "restore($restoreId): failed due to" }
 
@@ -152,11 +158,14 @@ object ProtoBackupImport : ProtoBackupBase() {
         id: String,
         sourceStream: InputStream,
         flags: BackupFlags,
+        isSync: Boolean,
     ): ValidationResult {
         val backupString =
             sourceStream
                 .source()
-                .gzip()
+                .run {
+                    if (!isSync) gzip() else this
+                }
                 .buffer()
                 .use { it.readByteArray() }
         val backup = parser.decodeFromByteArray(Backup.serializer(), backupString)
@@ -235,6 +244,17 @@ object ProtoBackupImport : ProtoBackupBase() {
             """.trimIndent()
         }
 
+        if (isSync) {
+            transaction {
+                MangaTable.update({ MangaTable.isSyncing eq true}) {
+                    it[isSyncing] = false
+                }
+                ChapterTable.update({ ChapterTable.isSyncing eq true}) {
+                    it[isSyncing] = false
+                }
+            }
+        }
+
         updateRestoreState(id, BackupRestoreState.Success)
 
         return validationResult

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt

@@ -73,6 +73,7 @@ object BackupMangaHandler {
                         dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds,
                         viewer = 0, // not supported in Tachidesk
                         updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]),
+                        version = mangaRow[MangaTable.version],
                     )
 
                 val mangaId = mangaRow[MangaTable.id].value
@@ -108,6 +109,7 @@ object BackupMangaHandler {
                                     it.uploadDate,
                                     it.chapterNumber,
                                     chapters.size - it.index,
+                                    it.version
                                 ).apply {
                                     if (flags.includeClientData) {
                                         this.meta = chapterToMeta[it.id] ?: emptyMap()
@@ -230,6 +232,8 @@ object BackupMangaHandler {
                                 it[inLibrary] = manga.favorite
 
                                 it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
+
+                                it[version] = manga.version
                             }.value
                     } else {
                         val dbMangaId = dbManga[MangaTable.id].value
@@ -249,6 +253,8 @@ object BackupMangaHandler {
                             it[inLibrary] = manga.favorite || dbManga[inLibrary]
 
                             it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
+
+                            it[version] = manga.version
                         }
 
                         dbMangaId
@@ -337,6 +343,8 @@ object BackupMangaHandler {
                             this[ChapterTable.lastReadAt] =
                                 historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
                         }
+
+                        this[ChapterTable.version] = chapter.version
                     }.map { it[ChapterTable.id].value }
             } else {
                 emptyList()

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt

@@ -19,6 +19,8 @@ data class BackupChapter(
     // chapterNumber is called number is 1.x
     @ProtoNumber(9) var chapterNumber: Float = 0F,
     @ProtoNumber(10) var sourceOrder: Int = 0,
+    // syncyomi
+    @ProtoNumber(12) var version: Long = 0,
     // suwayomi
     @ProtoNumber(9000) var meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt

@@ -34,6 +34,8 @@ data class BackupManga(
     @ProtoNumber(103) var viewer_flags: Int? = null,
     @ProtoNumber(104) var history: List<BackupHistory> = emptyList(),
     @ProtoNumber(105) var updateStrategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE,
+    // syncyomi
+    @ProtoNumber(109) var version: Long = 0,
     // suwayomi
     @ProtoNumber(9000) var meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt

@@ -38,6 +38,7 @@ data class ChapterDataClass(
     val pageCount: Int = -1,
     /** total chapter count, used to calculate if there's a next and prev chapter */
     val chapterCount: Int? = null,
+    val version: Long = 0,
     /** used to store client specific values */
     val meta: Map<String, String> = emptyMap(),
 ) {

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt

@@ -43,6 +43,7 @@ data class MangaDataClass(
     val age: Long? = if (lastFetchedAt == null) 0 else Instant.now().epochSecond.minus(lastFetchedAt),
     val chaptersAge: Long? = if (chaptersLastFetchedAt == null) null else Instant.now().epochSecond.minus(chaptersLastFetchedAt),
     val trackers: List<MangaTrackerDataClass>? = null,
+    val version: Long = 0,
 ) {
     override fun toString(): String = "\"$title\" (id= $id) (sourceId= $sourceId)"
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt

@@ -41,6 +41,9 @@ object ChapterTable : IntIdTable() {
     val manga = reference("manga", MangaTable, ReferenceOption.CASCADE)
 
     val koreaderHash = varchar("koreader_hash", 32).nullable()
+
+    val version = long("version").default(0)
+    val isSyncing = bool("is_syncing").default(false)
 }
 
 fun ChapterTable.toDataClass(
@@ -82,4 +85,5 @@ fun ChapterTable.toDataClass(
         } else {
             emptyMap()
         },
+    version = chapterEntry[version],
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt

@@ -46,6 +46,9 @@ object MangaTable : IntIdTable() {
     val chaptersLastFetchedAt = long("chapters_last_fetched_at").default(0)
 
     val updateStrategy = varchar("update_strategy", 256).default(UpdateStrategy.ALWAYS_UPDATE.name)
+
+    val version = long("version").default(0)
+    val isSyncing = bool("is_syncing").default(false)
 }
 
 fun MangaTable.toDataClass(
@@ -76,6 +79,7 @@ fun MangaTable.toDataClass(
     lastFetchedAt = mangaEntry[lastFetchedAt],
     chaptersLastFetchedAt = mangaEntry[chaptersLastFetchedAt],
     updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy]),
+    version = mangaEntry[version],
 )
 
 enum class MangaStatus(

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0053_SyncYomi.kt

@@ -0,0 +1,29 @@
+package suwayomi.tachidesk.server.database.migration
+
+import de.neonew.exposed.migrations.helpers.SQLMigration
+
+@Suppress("ClassName", "unused")
+class M0053_SyncYomi : SQLMigration() {
+    override val sql = """
+        ALTER TABLE MANGA ADD COLUMN VERSION BIGINT DEFAULT 0;
+        ALTER TABLE MANGA ADD COLUMN IS_SYNCING BOOLEAN DEFAULT 0;
+
+        ALTER TABLE CHAPTER ADD COLUMN VERSION BIGINT DEFAULT 0;
+        ALTER TABLE CHAPTER ADD COLUMN IS_SYNCING BOOLEAN DEFAULT 0;
+                
+        CREATE TRIGGER update_manga_version 
+        AFTER UPDATE ON MANGA
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateMangaVersionTrigger";
+        
+        CREATE TRIGGER update_chapter_and_manga_version
+        AFTER UPDATE ON CHAPTER
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateChapterAndMangaVersionTrigger";
+        
+        CREATE TRIGGER insert_manga_category_update_version
+        AFTER INSERT ON CATEGORYMANGA
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.InsertMangaCategoryUpdateVersionTrigger";
+        """.trimIndent()
+}

server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt

@@ -0,0 +1,84 @@
+package suwayomi.tachidesk.server.database.trigger
+
+import org.h2.tools.TriggerAdapter
+import java.sql.Connection
+import java.sql.ResultSet
+
+@Suppress("unused")
+class UpdateMangaVersionTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet,
+        newRow: ResultSet,
+    ) {
+        val isSyncing = newRow.getBoolean("is_syncing")
+        val hasChanged = oldRow.getString("url") != newRow.getString("url") ||
+            oldRow.getString("description") != newRow.getString("description") ||
+            oldRow.getBoolean("in_library") != newRow.getBoolean("in_library")
+
+        if (!isSyncing && hasChanged) {
+            val id = newRow.getInt("id")
+
+            conn.prepareStatement(
+                "UPDATE MANGA SET version = version + 1 WHERE id = ?",
+            ).use {
+                it.setInt(1, id)
+                it.executeUpdate()
+            }
+        }
+    }
+}
+
+@Suppress("unused")
+class UpdateChapterAndMangaVersionTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet,
+        newRow: ResultSet,
+    ) {
+        val isSyncing = newRow.getBoolean("is_syncing")
+        val hasChanged = oldRow.getBoolean("read") != newRow.getBoolean("read") ||
+            oldRow.getBoolean("bookmark") != newRow.getBoolean("bookmark") ||
+            oldRow.getInt("last_page_read") != newRow.getInt("last_page_read")
+
+        if (!isSyncing && hasChanged) {
+            val chapterId = newRow.getInt("id")
+            val mangaId = newRow.getInt("manga")
+
+            conn.prepareStatement(
+                "UPDATE CHAPTER SET version = version + 1 WHERE id = ?",
+            ).use {
+                it.setInt(1, chapterId)
+                it.executeUpdate()
+            }
+
+            conn.prepareStatement(
+                "UPDATE MANGA SET version = version + 1 WHERE id = ? AND (SELECT is_syncing FROM MANGA WHERE id = ?) = FALSE",
+            ).use {
+                it.setInt(1, mangaId)
+                it.setInt(2, mangaId)
+                it.executeUpdate()
+            }
+        }
+    }
+}
+
+@Suppress("unused")
+class InsertMangaCategoryUpdateVersionTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet?,
+        newRow: ResultSet,
+    ) {
+        val mangaId = newRow.getInt("manga")
+
+        conn.prepareStatement(
+            "UPDATE MANGA SET version = version + 1 WHERE id = ? AND (SELECT is_syncing FROM MANGA WHERE id = ?) = FALSE",
+        ).use {
+            it.setInt(1, mangaId)
+            it.setInt(2, mangaId)
+            it.executeUpdate()
+        }
+    }
+}
+