Implement SyncYomi (#1813)

* Implement SyncYomi

* Add ability to select what to sync

* Properly fix default category bug

* Add periodic sync

* Add PostgreSQL support

* Deschedule previous task

* Check if SyncYomi is enabled in syncData function

* Don't allow multiple syncs at the same time

* Convert SyncYomiSyncService to object

* Make startSync non-suspend

* Return a result from startSync

* Sync before library update

* Improvements

* Use NetworkHelper client

* Lint

* Use measureTime

* Database improvements
- Move entire sync operation into a single transaction
- Stop loading all manga to memory

* Revert "Database improvements"

This reverts commit bee8d214c3.

* Actual database improvements

* Remove runBlocking

* Remove title check

* Update updateNonFavorites function

* Update timeout code

* Improve PostgreSQL query

* Create lastSyncState variable

* Create lastSyncStatus query

* Convert lastSyncState to StateFlow

* Create lastSyncStatusChange subscription

* Replace backupRestoreStatus with backupRestoreId

* Add startDate and endDate

* Add logs for sync start and end

* Handle all errors in syncData

* Change category restore function to match Mihon's behavior

* Fix comment

* Remove duplicate BackupMangaHandler.backup call

* Remove duplicated log

* Rename subscription to syncStatusChanged

* Use same flags for restoring

* Update syncInterval config to use DurationSetting

* Update sync scheduling logic

* Reorder conditions to reduce database calls

* Prevent deleted ghost chapters from reappearing during sync
jobobby04/TachiyomiSY#1575

* Improve sync merging categories
jobobby04/TachiyomiSY#1559

* Make columns not null

* Improve H2 triggers

* Add documentation

docs/Configuring-Suwayomi‐Server.md

@@ -276,6 +276,28 @@ server.useHikariConnectionPool = true
 - `server.databasePassword` the username with which to authenticate at the PostgreSQL instance.
 - `server.useHikariConnectionPool` use Hikari Connection Pool to connect to the database.
 
+### SyncYomi
+```
+server.syncYomiEnabled = false
+server.syncYomiHost = ""
+server.syncYomiApiKey = ""
+server.syncDataManga = true
+server.syncDataChapters = true
+server.syncDataTracking = true
+server.syncDataHistory = true
+server.syncDataCategories = true
+server.syncInterval = "0s"
+```
+- `server.syncYomiEnabled` controls whether SyncYomi is enabled.
+- `server.syncYomiHost` base URL of the SyncYomi server instance. e.g. `http://localhost:8282`
+- `server.syncYomiApiKey` API key to authenticate with SyncYomi. You must use the same API key in both Suwayomi and SyncYomi.
+- `server.syncDataManga` enables syncing manga.
+- `server.syncDataChapters` enables syncing chapters.
+- `server.syncDataTracking` enables syncing tracking data.
+- `server.syncDataHistory` enables syncing reading history.
+- `server.syncDataCategories` enables syncing categories.
+- `server.syncInterval` interval between automatic sync operations. Use `0s` to disable.
+
 **Note:** The example [docker-compose.yml file](https://github.com/Suwayomi/Suwayomi-Server-docker/blob/main/docker-compose.yml) contains everything you need to get started with Suwayomi+PostgreSQL. Please be aware that PostgreSQL support is currently still in beta.
 
 **Note:** These settings are excluded from backups, so a backup can be used to easily switch database installations by setting up the connection first, then restoring the backup.

server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt

@@ -1038,7 +1038,68 @@ class ServerConfig(
         description = "Enable the WebView via CEF (Chromium)"
     )
 
+    val syncYomiEnabled: MutableStateFlow<Boolean> by BooleanSetting(
+        protoNumber = 87,
+        defaultValue = false,
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = true
+    )
 
+    val syncYomiHost: MutableStateFlow<String> by StringSetting(
+        protoNumber = 88,
+        defaultValue = "",
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = true,
+    )
+
+    val syncYomiApiKey: MutableStateFlow<String> by StringSetting(
+        protoNumber = 89,
+        defaultValue = "",
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = false,
+    )
+
+    val syncDataManga: MutableStateFlow<Boolean> by BooleanSetting(
+        protoNumber = 90,
+        defaultValue = true,
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = true,
+    )
+
+    val syncDataChapters: MutableStateFlow<Boolean> by BooleanSetting(
+        protoNumber = 91,
+        defaultValue = true,
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = true,
+    )
+
+    val syncDataTracking: MutableStateFlow<Boolean> by BooleanSetting(
+        protoNumber = 92,
+        defaultValue = true,
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = true,
+    )
+
+    val syncDataHistory: MutableStateFlow<Boolean> by BooleanSetting(
+        protoNumber = 93,
+        defaultValue = true,
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = true,
+    )
+
+    val syncDataCategories: MutableStateFlow<Boolean> by BooleanSetting(
+        protoNumber = 94,
+        defaultValue = true,
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = true,
+    )
+
+    val syncInterval: MutableStateFlow<Duration> by DurationSetting(
+        protoNumber = 95,
+        defaultValue = 0.seconds,
+        group = SettingGroup.SYNCYOMI,
+        privacySafe = true,
+    )
 
     /** ****************************************************************** **/
     /**                                                                    **/

server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt

@@ -18,6 +18,7 @@ enum class SettingGroup(
     OPDS("OPDS"),
     KOREADER_SYNC("KOReader sync"),
     WEB_VIEW("WebView"),
+    SYNCYOMI("SyncYomi")
     ;
 
     override fun toString(): String = value

server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt

@@ -0,0 +1,519 @@
+package suwayomi.tachidesk.global.impl.sync
+
+import android.app.Application
+import android.content.Context
+import io.github.oshai.kotlinlogging.KotlinLogging
+import kotlinx.coroutines.DelicateCoroutinesApi
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+import kotlinx.coroutines.flow.combine
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.protobuf.ProtoBuf
+import org.jetbrains.exposed.v1.core.and
+import org.jetbrains.exposed.v1.core.eq
+import org.jetbrains.exposed.v1.jdbc.selectAll
+import org.jetbrains.exposed.v1.jdbc.transactions.transaction
+import org.jetbrains.exposed.v1.jdbc.update
+import suwayomi.tachidesk.graphql.types.StartSyncResult
+import suwayomi.tachidesk.manga.impl.Category
+import suwayomi.tachidesk.manga.impl.Library.handleMangaThumbnail
+import suwayomi.tachidesk.manga.impl.backup.BackupFlags
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport
+import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler
+import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
+import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
+import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
+import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass
+import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
+import suwayomi.tachidesk.manga.model.table.CategoryMangaTable
+import suwayomi.tachidesk.manga.model.table.CategoryTable
+import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.MangaTable
+import suwayomi.tachidesk.manga.model.table.toDataClass
+import suwayomi.tachidesk.server.serverConfig
+import suwayomi.tachidesk.util.HAScheduler
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import kotlin.time.Clock
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.Instant
+import kotlin.time.measureTime
+
+@Serializable
+data class SyncData(
+    val backup: Backup? = null,
+)
+
+object SyncManager {
+    private val syncPreferences = Injekt.get<Application>().getSharedPreferences("sync", Context.MODE_PRIVATE)
+    private val logger = KotlinLogging.logger {}
+
+    private var currentTaskId: String? = null
+    private val syncMutex = Mutex()
+
+    private val _lastSyncState: MutableStateFlow<SyncState?> = MutableStateFlow(null)
+    val lastSyncState: StateFlow<SyncState?> = _lastSyncState.asStateFlow()
+
+    @OptIn(DelicateCoroutinesApi::class)
+    fun scheduleSyncTask() {
+        serverConfig.subscribeTo(
+            combine(
+                serverConfig.syncYomiEnabled,
+                serverConfig.syncInterval,
+            ) { enabled, interval -> Pair(enabled, interval) },
+            { (enabled, interval) ->
+                currentTaskId?.let { HAScheduler.deschedule(it) }
+
+                currentTaskId =
+                    if (enabled && interval > 0.seconds) {
+                        val lastSyncDate =
+                            syncPreferences
+                                .getLong("last_scheduled_sync", 0L)
+                                .takeIf { it != 0L }
+                                ?.let { Instant.fromEpochMilliseconds(it) }
+
+                        if (lastSyncDate == null) {
+                            syncPreferences
+                                .edit()
+                                .putLong("last_scheduled_sync", Clock.System.now().toEpochMilliseconds())
+                                .apply()
+                        }
+
+                        val delay =
+                            if (lastSyncDate != null) {
+                                ((interval) - (Clock.System.now() - lastSyncDate)).coerceAtLeast(0.seconds)
+                            } else {
+                                interval
+                            }
+
+                        HAScheduler.schedule(
+                            {
+                                startSync(periodic = true)
+
+                                syncPreferences
+                                    .edit()
+                                    .putLong("last_scheduled_sync", Clock.System.now().toEpochMilliseconds())
+                                    .apply()
+                            },
+                            interval = interval.inWholeMilliseconds,
+                            delay = delay.inWholeMilliseconds,
+                            name = "sync",
+                        )
+                    } else {
+                        syncPreferences
+                            .edit()
+                            .remove("last_scheduled_sync")
+                            .apply()
+                        null
+                    }
+            },
+            ignoreInitialValue = false,
+        )
+    }
+
+    @OptIn(DelicateCoroutinesApi::class)
+    fun startSync(periodic: Boolean = false): StartSyncResult {
+        if (!serverConfig.syncYomiEnabled.value) {
+            return StartSyncResult.SYNC_DISABLED
+        }
+
+        if (!syncMutex.tryLock()) {
+            return StartSyncResult.SYNC_IN_PROGRESS
+        }
+
+        GlobalScope.launch {
+            try {
+                syncData(periodic)
+            } finally {
+                syncMutex.unlock()
+            }
+        }
+
+        return StartSyncResult.SUCCESS
+    }
+
+    suspend fun ensureSync() {
+        if (!serverConfig.syncYomiEnabled.value) {
+            return
+        }
+
+        if (syncMutex.tryLock()) {
+            // there is no ongoing sync, so start one
+            try {
+                syncData()
+            } finally {
+                syncMutex.unlock()
+            }
+        } else {
+            // wait for the ongoing sync to finish
+            syncMutex.withLock {}
+        }
+    }
+
+    private suspend fun syncData(periodic: Boolean = false) {
+        val startInstant = Clock.System.now()
+        _lastSyncState.value = SyncState.Started(startInstant)
+
+        try {
+            logger.info {
+                if (periodic) {
+                    "Starting periodic sync"
+                } else {
+                    "Starting manual sync"
+                }
+            }
+
+            transaction {
+                MangaTable.update({ MangaTable.isSyncing eq true }) {
+                    it[isSyncing] = false
+                }
+                ChapterTable.update({ ChapterTable.isSyncing eq true }) {
+                    it[isSyncing] = false
+                }
+                CategoryTable.update({ CategoryTable.isSyncing eq true }) {
+                    it[isSyncing] = false
+                }
+            }
+
+            val backupFlags =
+                BackupFlags(
+                    includeManga = serverConfig.syncDataManga.value,
+                    includeCategories = serverConfig.syncDataCategories.value,
+                    includeChapters = serverConfig.syncDataChapters.value,
+                    includeTracking = serverConfig.syncDataTracking.value,
+                    includeHistory = serverConfig.syncDataHistory.value,
+                    includeClientData = false,
+                    includeServerSettings = false,
+                )
+
+            _lastSyncState.value = SyncState.CreatingBackup(startInstant)
+            val backupMangas = BackupMangaHandler.backup(backupFlags)
+            val backup =
+                Backup(
+                    backupMangas,
+                    BackupCategoryHandler.backup(backupFlags).filter { it.name != Category.DEFAULT_CATEGORY_NAME },
+                    BackupSourceHandler.backup(backupMangas, backupFlags),
+                    emptyMap(),
+                    null,
+                )
+
+            val syncData =
+                SyncData(
+                    backup = backup,
+                )
+
+            val remoteBackup =
+                SyncYomiSyncService.doSync(syncData, startInstant) {
+                    _lastSyncState.value = it
+                }
+
+            if (remoteBackup == null) {
+                logger.debug { "Skip restore due to network issues" }
+                finishWithError(startInstant, "Network error", periodic)
+                return
+            }
+
+            if (remoteBackup === syncData.backup) {
+                // nothing changed
+                logger.debug { "Skip restore due to remote was overwrite from local" }
+                finishWithSuccess(startInstant, periodic)
+                return
+            }
+
+            // Stop the sync early if the remote backup is null or empty
+            if (remoteBackup.backupManga.isEmpty() && remoteBackup.backupCategories.isEmpty() && remoteBackup.backupSources.isEmpty()) {
+                logger.error { "No data found on remote server." }
+                finishWithError(startInstant, "No data found on remote server.", periodic)
+                return
+            }
+
+            val isLibraryEmpty =
+                transaction {
+                    MangaTable
+                        .selectAll()
+                        .where { MangaTable.inLibrary eq true }
+                        .empty()
+                }
+
+            // Check if it's first sync based on lastSyncTimestamp
+            if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && !isLibraryEmpty) {
+                // It's first sync no need to restore data. (just update remote data)
+                finishWithSuccess(startInstant, periodic)
+                return
+            }
+
+            val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup)
+            updateNonFavorites(nonFavorites)
+
+            val newSyncData =
+                backup.copy(
+                    backupManga = filteredFavorites,
+                    backupCategories = remoteBackup.backupCategories,
+                    backupSources = remoteBackup.backupSources,
+                )
+
+            val hasMangaChanges = filteredFavorites.isNotEmpty()
+            val hasCategoryChanges = remoteBackup.backupCategories != backup.backupCategories
+            val hasSourceChanges = remoteBackup.backupSources != backup.backupSources
+
+            if (!hasMangaChanges && !hasCategoryChanges && !hasSourceChanges) {
+                // update the sync timestamp
+                finishWithSuccess(startInstant, periodic)
+                return
+            }
+
+            if (serverConfig.syncDataCategories.value) {
+                val mergedUids = newSyncData.backupCategories.map { it.uid }.toSet()
+                val mergedNames = newSyncData.backupCategories.map { it.name }.toSet()
+                val localCategories = Category.getCategoryList().filterNot { it.default } // Exclude system category
+                val categoriesToDelete =
+                    localCategories.filter {
+                        it.uid !in mergedUids && it.name !in mergedNames
+                    }
+                if (categoriesToDelete.isNotEmpty()) {
+                    transaction {
+                        categoriesToDelete.forEach {
+                            Category.removeCategory(it.id)
+                        }
+                    }
+                }
+            }
+
+            val backupStream = ProtoBuf.encodeToByteArray(Backup.serializer(), newSyncData).inputStream()
+            val restoreId =
+                ProtoBackupImport.restore(
+                    sourceStream = backupStream,
+                    flags = backupFlags,
+                    isSync = true,
+                )
+            _lastSyncState.value = SyncState.Restoring(startInstant, restoreId)
+
+            ProtoBackupImport.notifyFlow.first {
+                val restoreState = ProtoBackupImport.getRestoreState(restoreId)
+
+                restoreState == ProtoBackupImport.BackupRestoreState.Success ||
+                    restoreState == ProtoBackupImport.BackupRestoreState.Failure
+            }
+
+            // update the sync timestamp
+            finishWithSuccess(startInstant, periodic)
+        } catch (e: Throwable) {
+            logger.error { "Error syncing: ${e.message}" }
+            finishWithError(startInstant, "${e::class.qualifiedName}: ${e.message}", periodic)
+        }
+    }
+
+    private fun finishWithSuccess(
+        startInstant: Instant,
+        periodic: Boolean,
+    ) {
+        syncPreferences
+            .edit()
+            .putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
+            .apply()
+        _lastSyncState.value = SyncState.Success(startInstant)
+
+        logger.info {
+            if (periodic) {
+                "Periodic sync completed successfully"
+            } else {
+                "Manual sync completed successfully"
+            }
+        }
+    }
+
+    private fun finishWithError(
+        startInstant: Instant,
+        message: String,
+        periodic: Boolean,
+    ) {
+        _lastSyncState.value = SyncState.Error(startInstant, message)
+
+        logger.info {
+            if (periodic) {
+                "Periodic sync failed: $message"
+            } else {
+                "Manual sync failed: $message"
+            }
+        }
+    }
+
+    private fun isMangaDifferent(
+        localManga: MangaDataClass,
+        remoteManga: BackupManga,
+    ): Boolean {
+        if (localManga.version != remoteManga.version) {
+            return true
+        }
+
+        val localChapters =
+            transaction {
+                ChapterTable
+                    .selectAll()
+                    .where { ChapterTable.manga eq localManga.id }
+                    .map { ChapterTable.toDataClass(it) }
+            }
+
+        if (areChaptersDifferent(localChapters, remoteManga.chapters)) {
+            return true
+        }
+
+        val localCategories =
+            transaction {
+                CategoryMangaTable
+                    .innerJoin(CategoryTable)
+                    .selectAll()
+                    .where { CategoryMangaTable.manga eq localManga.id }
+                    .map { it[CategoryTable.order] }
+            }
+
+        return localCategories.toSet() != remoteManga.categories.toSet()
+    }
+
+    private fun areChaptersDifferent(
+        localChapters: List<ChapterDataClass>,
+        remoteChapters: List<BackupChapter>,
+    ): Boolean {
+        val localChapterMap = localChapters.associateBy { it.url }
+        val remoteChapterMap = remoteChapters.associateBy { it.url }
+
+        if (localChapterMap.size != remoteChapterMap.size) {
+            return true
+        }
+
+        for ((url, localChapter) in localChapterMap) {
+            val remoteChapter = remoteChapterMap[url]
+
+            // If a matching remote chapter doesn't exist, or the version numbers are different, consider them different
+            if (remoteChapter == null || localChapter.version != remoteChapter.version) {
+                return true
+            }
+        }
+
+        return false
+    }
+
+    private fun filterFavoritesAndNonFavorites(backup: Backup): Pair<List<BackupManga>, List<BackupManga>> {
+        val favorites = mutableListOf<BackupManga>()
+        val nonFavorites = mutableListOf<BackupManga>()
+
+        val elapsedTime =
+            measureTime {
+                logger.debug { "Starting to filter favorites and non-favorites from backup data." }
+
+                backup.backupManga.forEach { remoteManga ->
+                    val localManga =
+                        transaction {
+                            MangaTable
+                                .selectAll()
+                                .where {
+                                    (MangaTable.sourceReference eq remoteManga.source) and
+                                        (MangaTable.url eq remoteManga.url)
+                                }.limit(1)
+                                .map { MangaTable.toDataClass(it) }
+                                .firstOrNull()
+                        }
+
+                    when {
+                        // Checks if the manga is in favorites and needs updating or adding
+                        remoteManga.favorite -> {
+                            if (localManga == null || isMangaDifferent(localManga, remoteManga)) {
+                                logger.debug { "Adding to favorites: ${remoteManga.title}" }
+                                favorites.add(remoteManga)
+                            } else {
+                                logger.debug { "Already up-to-date favorite: ${remoteManga.title}" }
+                            }
+                        }
+
+                        // Handle non-favorites
+                        !remoteManga.favorite -> {
+                            logger.debug { "Adding to non-favorites: ${remoteManga.title}" }
+                            nonFavorites.add(remoteManga)
+                        }
+                    }
+                }
+            }
+
+        logger.debug {
+            "Filtering completed in $elapsedTime. Favorites found: ${favorites.size}, Non-favorites found: ${nonFavorites.size}"
+        }
+
+        return Pair(favorites, nonFavorites)
+    }
+
+    private fun updateNonFavorites(nonFavorites: List<BackupManga>) {
+        nonFavorites.forEach { nonFavorite ->
+            val localManga =
+                transaction {
+                    MangaTable
+                        .selectAll()
+                        .where {
+                            (MangaTable.sourceReference eq nonFavorite.source) and
+                                (MangaTable.url eq nonFavorite.url)
+                        }.limit(1)
+                        .map { MangaTable.toDataClass(it) }
+                        .firstOrNull()
+                }
+
+            if (localManga != null) {
+                if (localManga.inLibrary != nonFavorite.favorite) {
+                    transaction {
+                        MangaTable.update({ MangaTable.id eq localManga.id }) {
+                            it[inLibrary] = nonFavorite.favorite
+                        }
+                    }.apply {
+                        handleMangaThumbnail(localManga.id, nonFavorite.favorite)
+                    }
+                }
+            }
+        }
+    }
+
+    sealed class SyncState(
+        open val startDate: Instant,
+    ) {
+        data class Started(
+            override val startDate: Instant,
+        ) : SyncState(startDate)
+
+        data class CreatingBackup(
+            override val startDate: Instant,
+        ) : SyncState(startDate)
+
+        data class Downloading(
+            override val startDate: Instant,
+        ) : SyncState(startDate)
+
+        data class Merging(
+            override val startDate: Instant,
+        ) : SyncState(startDate)
+
+        data class Uploading(
+            override val startDate: Instant,
+        ) : SyncState(startDate)
+
+        data class Restoring(
+            override val startDate: Instant,
+            val restoreId: String,
+        ) : SyncState(startDate)
+
+        data class Success(
+            override val startDate: Instant,
+            val endDate: Instant = Clock.System.now(),
+        ) : SyncState(startDate)
+
+        data class Error(
+            override val startDate: Instant,
+            val message: String,
+            val endDate: Instant = Clock.System.now(),
+        ) : SyncState(startDate)
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt

@@ -0,0 +1,517 @@
+package suwayomi.tachidesk.global.impl.sync
+
+import android.app.Application
+import android.content.Context
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.NetworkHelper
+import eu.kanade.tachiyomi.network.PUT
+import eu.kanade.tachiyomi.network.await
+import io.github.oshai.kotlinlogging.KotlinLogging
+import io.javalin.http.HttpStatus
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.protobuf.ProtoBuf
+import okhttp3.Headers
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.RequestBody.Companion.toRequestBody
+import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga
+import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource
+import suwayomi.tachidesk.server.serverConfig
+import uy.kohesive.injekt.Injekt
+import uy.kohesive.injekt.api.get
+import uy.kohesive.injekt.injectLazy
+import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.Instant
+
+object SyncYomiSyncService {
+    private val syncPreferences = Injekt.get<Application>().getSharedPreferences("sync", Context.MODE_PRIVATE)
+
+    private val network: NetworkHelper by injectLazy()
+    private val logger = KotlinLogging.logger {}
+
+    private class SyncYomiException(
+        message: String?,
+    ) : Exception(message)
+
+    suspend fun doSync(
+        syncData: SyncData,
+        startDate: Instant,
+        setSyncState: (SyncManager.SyncState) -> Unit,
+    ): Backup? {
+        setSyncState(SyncManager.SyncState.Downloading(startDate))
+        val (remoteData, etag) = pullSyncData()
+
+        val finalSyncData =
+            if (remoteData != null) {
+                require(etag.isNotEmpty()) { "ETag should never be empty if remote data is not null" }
+                logger.debug { "Try update remote data with ETag($etag)" }
+                setSyncState(SyncManager.SyncState.Merging(startDate))
+                mergeSyncData(syncData, remoteData)
+            } else {
+                // init or overwrite remote data
+                logger.debug { "Try overwrite remote data with ETag($etag)" }
+                syncData
+            }
+
+        if (finalSyncData.backup != null) {
+            setSyncState(SyncManager.SyncState.Uploading(startDate))
+        }
+        pushSyncData(finalSyncData, etag)
+        return finalSyncData.backup
+    }
+
+    private suspend fun pullSyncData(): Pair<SyncData?, String> {
+        val host = serverConfig.syncYomiHost.value
+        val apiKey = serverConfig.syncYomiApiKey.value
+        val downloadUrl = "$host/api/sync/content"
+
+        val headersBuilder = Headers.Builder().add("X-API-Token", apiKey)
+        val lastETag = syncPreferences.getString("last_sync_etag", "") ?: ""
+        if (lastETag != "") {
+            headersBuilder.add("If-None-Match", lastETag)
+        }
+        val headers = headersBuilder.build()
+
+        val downloadRequest =
+            GET(
+                url = downloadUrl,
+                headers = headers,
+            )
+
+        val response = network.client.newCall(downloadRequest).await()
+
+        if (response.code == HttpStatus.NOT_MODIFIED.code) {
+            // not modified
+            require(lastETag.isNotEmpty())
+            logger.info { "Remote server not modified" }
+            return Pair(null, lastETag)
+        } else if (response.code == HttpStatus.NOT_FOUND.code) {
+            // maybe got deleted from remote
+            return Pair(null, "")
+        }
+
+        if (response.isSuccessful) {
+            val newETag =
+                response.headers["ETag"]
+                    ?.takeIf { it.isNotEmpty() } ?: throw SyncYomiException("Missing ETag")
+
+            val byteArray =
+                response.body.byteStream().use {
+                    return@use it.readBytes()
+                }
+
+            return try {
+                val backup = ProtoBuf.decodeFromByteArray(Backup.serializer(), byteArray)
+                return Pair(SyncData(backup = backup), newETag)
+            } catch (_: SerializationException) {
+                logger.info { "Bad content responsed from server" }
+                // the body is invalid
+                // return default value so we can overwrite it
+                Pair(null, "")
+            }
+        } else {
+            val responseBody = response.body.string()
+            logger.error { "SyncError: $responseBody" }
+            throw SyncYomiException("Failed to download sync data: $responseBody")
+        }
+    }
+
+    private suspend fun pushSyncData(
+        syncData: SyncData,
+        eTag: String,
+    ) {
+        val backup = syncData.backup ?: return
+
+        val host = serverConfig.syncYomiHost.value
+        val apiKey = serverConfig.syncYomiApiKey.value
+        val uploadUrl = "$host/api/sync/content"
+
+        val headersBuilder = Headers.Builder().add("X-API-Token", apiKey)
+        if (eTag.isNotEmpty()) {
+            headersBuilder.add("If-Match", eTag)
+        }
+        val headers = headersBuilder.build()
+
+        // Set timeout to 30 seconds
+        val timeout = 30.seconds
+        val client =
+            network.client
+                .newBuilder()
+                .connectTimeout(timeout)
+                .readTimeout(timeout)
+                .writeTimeout(timeout)
+                .build()
+
+        val byteArray = ProtoBuf.encodeToByteArray(Backup.serializer(), backup)
+        if (byteArray.isEmpty()) {
+            throw IllegalStateException("Empty backup error")
+        }
+        val body = byteArray.toRequestBody("application/octet-stream".toMediaType())
+
+        val uploadRequest =
+            PUT(
+                url = uploadUrl,
+                headers = headers,
+                body = body,
+            )
+
+        val response = client.newCall(uploadRequest).await()
+
+        if (response.isSuccessful) {
+            val newETag =
+                response.headers["ETag"]
+                    ?.takeIf { it.isNotEmpty() } ?: throw SyncYomiException("Missing ETag")
+            syncPreferences
+                .edit()
+                .putString("last_sync_etag", newETag)
+                .apply()
+            logger.debug { "SyncYomi sync completed" }
+        } else if (response.code == HttpStatus.PRECONDITION_FAILED.code) {
+            // other clients updated remote data, will try next time
+            logger.debug { "SyncYomi sync failed with 412" }
+        } else {
+            val responseBody = response.body.string()
+            logger.error { "SyncError: $responseBody" }
+        }
+    }
+
+    fun mergeSyncData(
+        localSyncData: SyncData,
+        remoteSyncData: SyncData,
+    ): SyncData {
+        val mergedCategoriesList =
+            mergeCategoriesLists(localSyncData.backup?.backupCategories, remoteSyncData.backup?.backupCategories)
+
+        val mergedMangaList =
+            mergeMangaLists(
+                localSyncData.backup?.backupManga,
+                remoteSyncData.backup?.backupManga,
+                localSyncData.backup?.backupCategories ?: emptyList(),
+                remoteSyncData.backup?.backupCategories ?: emptyList(),
+                mergedCategoriesList,
+            )
+
+        val mergedSourcesList =
+            mergeSourcesLists(localSyncData.backup?.backupSources, remoteSyncData.backup?.backupSources)
+
+        // Create the merged Backup object
+        val mergedBackup =
+            Backup(
+                backupManga = mergedMangaList,
+                backupCategories = mergedCategoriesList,
+                backupSources = mergedSourcesList,
+                meta = emptyMap(),
+                serverSettings = null,
+            )
+
+        // Create the merged SData object
+        return SyncData(
+            backup = mergedBackup,
+        )
+    }
+
+    private fun mergeMangaLists(
+        localMangaList: List<BackupManga>?,
+        remoteMangaList: List<BackupManga>?,
+        localCategories: List<BackupCategory>,
+        remoteCategories: List<BackupCategory>,
+        mergedCategories: List<BackupCategory>,
+    ): List<BackupManga> {
+        val localMangaListSafe = localMangaList.orEmpty()
+        val remoteMangaListSafe = remoteMangaList.orEmpty()
+
+        logger.debug { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" }
+
+        fun mangaCompositeKey(manga: BackupManga): String =
+            "${manga.source}|${manga.url}|${manga.title.lowercase().trim()}|${manga.author?.lowercase()?.trim()}"
+
+        // Create maps using composite keys
+        val localMangaMap = localMangaListSafe.associateBy { mangaCompositeKey(it) }
+        val remoteMangaMap = remoteMangaListSafe.associateBy { mangaCompositeKey(it) }
+
+        val localCategoriesMapByOrder = localCategories.associateBy { it.order }
+        val remoteCategoriesMapByOrder = remoteCategories.associateBy { it.order }
+        val mergedCategoriesMapByName = mergedCategories.associateBy { it.name }
+
+        fun updateCategories(
+            theManga: BackupManga,
+            theMap: Map<Int, BackupCategory>,
+        ): BackupManga =
+            theManga.copy(
+                categories =
+                    theManga.categories.mapNotNull {
+                        theMap[it]?.let { category ->
+                            mergedCategoriesMapByName[category.name]?.order
+                        }
+                    },
+            )
+
+        val lastSyncTime = syncPreferences.getLong("last_sync_timestamp", 0).milliseconds.inWholeSeconds
+
+        val mergedList =
+            (localMangaMap.keys + remoteMangaMap.keys).distinct().mapNotNull { compositeKey ->
+                val local = localMangaMap[compositeKey]
+                val remote = remoteMangaMap[compositeKey]
+
+                // New version comparison logic
+                when {
+                    local != null && remote == null -> {
+                        if (lastSyncTime == 0L || local.lastModifiedAt > lastSyncTime) {
+                            updateCategories(local, localCategoriesMapByOrder)
+                        } else {
+                            logger.debug { "Dropping local manga deleted on remote: ${local.title}." }
+                            null
+                        }
+                    }
+
+                    local == null && remote != null -> {
+                        if (lastSyncTime == 0L || remote.lastModifiedAt > lastSyncTime) {
+                            updateCategories(remote, remoteCategoriesMapByOrder)
+                        } else {
+                            logger.debug { "Dropping deleted remote manga: ${remote.title}." }
+                            null
+                        }
+                    }
+
+                    local != null && remote != null -> {
+                        // Compare versions to decide which manga to keep
+                        if (local.version >= remote.version) {
+                            logger.debug { "Keeping local version of ${local.title} with merged chapters." }
+                            updateCategories(
+                                local.copy(
+                                    chapters =
+                                        mergeChapters(
+                                            local.chapters,
+                                            remote.chapters,
+                                            lastSyncTime,
+                                            serverConfig.syncDataChapters.value,
+                                        ),
+                                ),
+                                localCategoriesMapByOrder,
+                            )
+                        } else {
+                            logger.debug { "Keeping remote version of ${remote.title} with merged chapters." }
+                            updateCategories(
+                                remote.copy(
+                                    chapters =
+                                        mergeChapters(
+                                            local.chapters,
+                                            remote.chapters,
+                                            lastSyncTime,
+                                            serverConfig.syncDataChapters.value,
+                                        ),
+                                ),
+                                remoteCategoriesMapByOrder,
+                            )
+                        }
+                    }
+
+                    else -> {
+                        null
+                    } // No manga found for key
+                }
+            }
+
+        // Counting favorites and non-favorites
+        val (favorites, nonFavorites) = mergedList.partition { it.favorite }
+
+        logger.debug {
+            "Merge completed. Total merged manga: ${mergedList.size}, Favorites: ${favorites.size}, Non-Favorites: ${nonFavorites.size}"
+        }
+
+        return mergedList
+    }
+
+    private fun mergeChapters(
+        localChapters: List<BackupChapter>,
+        remoteChapters: List<BackupChapter>,
+        lastSyncTime: Long,
+        syncingChapters: Boolean,
+    ): List<BackupChapter> {
+        if (!syncingChapters) {
+            return remoteChapters // If not syncing chapters, keep remote untouched
+        }
+
+        fun chapterCompositeKey(chapter: BackupChapter): String = "${chapter.url}|${chapter.name}|${chapter.chapterNumber}"
+
+        val localChapterMap = localChapters.associateBy { chapterCompositeKey(it) }
+        val remoteChapterMap = remoteChapters.associateBy { chapterCompositeKey(it) }
+
+        logger.debug { "Starting chapter merge. Local chapters: ${localChapters.size}, Remote chapters: ${remoteChapters.size}" }
+
+        // Merge both chapter maps based on version numbers
+        val mergedChapters =
+            (localChapterMap.keys + remoteChapterMap.keys).distinct().mapNotNull { compositeKey ->
+                val localChapter = localChapterMap[compositeKey]
+                val remoteChapter = remoteChapterMap[compositeKey]
+
+                logger.debug {
+                    "Processing chapter key: $compositeKey. Local chapter: ${localChapter != null}, Remote chapter: ${remoteChapter != null}"
+                }
+
+                when {
+                    localChapter != null && remoteChapter == null -> {
+                        if (lastSyncTime == 0L || localChapter.lastModifiedAt > lastSyncTime) {
+                            logger.debug { "Keeping local chapter: ${localChapter.name}." }
+                            localChapter
+                        } else {
+                            logger.debug { "Dropping local chapter deleted on remote: ${localChapter.name}." }
+                            null
+                        }
+                    }
+
+                    localChapter == null && remoteChapter != null -> {
+                        if (lastSyncTime == 0L || remoteChapter.lastModifiedAt > lastSyncTime) {
+                            logger.debug { "Taking remote chapter: ${remoteChapter.name}." }
+                            remoteChapter
+                        } else {
+                            logger.debug { "Dropping deleted remote chapter: ${remoteChapter.name}." }
+                            null
+                        }
+                    }
+
+                    localChapter != null && remoteChapter != null -> {
+                        // Use version number to decide which chapter to keep
+                        val chosenChapter =
+                            if (localChapter.version >= remoteChapter.version) {
+                                // If there mare more chapter on remote, local sourceOrder will need to be updated to maintain correct source order.
+                                if (localChapters.size < remoteChapters.size) {
+                                    localChapter.copy(sourceOrder = remoteChapter.sourceOrder)
+                                } else {
+                                    localChapter
+                                }
+                            } else {
+                                remoteChapter
+                            }
+                        logger.debug {
+                            "Merging chapter: ${chosenChapter.name}. Chosen version from: ${if (localChapter.version >= remoteChapter.version) "Local" else "Remote"}, Local version: ${localChapter.version}, Remote version: ${remoteChapter.version}."
+                        }
+                        chosenChapter
+                    }
+
+                    else -> {
+                        logger.debug { "No chapter found for composite key: $compositeKey. Skipping." }
+                        null
+                    }
+                }
+            }
+
+        logger.debug { "Chapter merge completed. Total merged chapters: ${mergedChapters.size}" }
+
+        return mergedChapters
+    }
+
+    private fun mergeCategoriesLists(
+        localCategoriesList: List<BackupCategory>?,
+        remoteCategoriesList: List<BackupCategory>?,
+    ): List<BackupCategory> {
+        if (localCategoriesList == null) return remoteCategoriesList ?: emptyList()
+        if (remoteCategoriesList == null) return localCategoriesList
+        val result = mutableListOf<BackupCategory>()
+        val processedLocals = mutableSetOf<BackupCategory>()
+
+        val localMapByUid = localCategoriesList.filter { it.uid != 0L }.associateBy { it.uid }
+        val localMapByName = localCategoriesList.associateBy { it.name }
+
+        val lastSyncTime = syncPreferences.getLong("last_sync_timestamp", 0)
+
+        remoteCategoriesList.forEach { remote ->
+            var localMatch: BackupCategory? = null
+
+            // 1. Try match by UID
+            if (remote.uid != 0L) {
+                localMatch = localMapByUid[remote.uid]
+            }
+
+            // 2. Try match by Name (fallback)
+            if (localMatch == null) {
+                localMatch = localMapByName[remote.name]
+            }
+
+            if (localMatch != null) {
+                processedLocals.add(localMatch)
+                // Conflict resolution
+                if (localMatch.version >= remote.version) {
+                    logger.debug { "Keeping local category: ${localMatch.name} (UID: ${localMatch.uid})" }
+                    result.add(localMatch)
+                } else {
+                    logger.debug { "Keeping remote category: ${remote.name} (UID: ${remote.uid})" }
+                    // Preserve Local UID if Remote was 0
+                    if (remote.uid == 0L) {
+                        remote.uid = localMatch.uid
+                    }
+                    result.add(remote)
+                }
+            } else {
+                val remoteModifiedTimeMillis = remote.lastModifiedAt.seconds.inWholeMilliseconds
+                if (lastSyncTime == 0L || remoteModifiedTimeMillis > lastSyncTime) {
+                    logger.debug { "Adding new remote category: ${remote.name} (UID: ${remote.uid})" }
+                    result.add(remote)
+                } else {
+                    logger.debug { "Dropping deleted remote category: ${remote.name} (UID: ${remote.uid})" }
+                }
+            }
+        }
+
+        // Add remaining Local Categories
+        localCategoriesList.forEach { local ->
+            if (local !in processedLocals) {
+                val localModifiedTimeMillis = local.lastModifiedAt.seconds.inWholeMilliseconds
+                if (lastSyncTime == 0L || localModifiedTimeMillis > lastSyncTime) {
+                    logger.debug { "Keeping local only category: ${local.name} (UID: ${local.uid})" }
+                    result.add(local)
+                } else {
+                    logger.debug { "Dropping local category deleted on remote: ${local.name} (UID: ${local.uid})" }
+                }
+            }
+        }
+
+        return result.sortedBy { it.order }
+    }
+
+    private fun mergeSourcesLists(
+        localSources: List<BackupSource>?,
+        remoteSources: List<BackupSource>?,
+    ): List<BackupSource> {
+        // Create maps using sourceId as key
+        val localSourceMap = localSources?.associateBy { it.sourceId } ?: emptyMap()
+        val remoteSourceMap = remoteSources?.associateBy { it.sourceId } ?: emptyMap()
+
+        logger.debug { "Starting source merge. Local sources: ${localSources?.size}, Remote sources: ${remoteSources?.size}" }
+
+        // Merge both source maps
+        val mergedSources =
+            (localSourceMap.keys + remoteSourceMap.keys).distinct().mapNotNull { sourceId ->
+                val localSource = localSourceMap[sourceId]
+                val remoteSource = remoteSourceMap[sourceId]
+
+                logger.debug {
+                    "Processing source ID: $sourceId. Local source: ${localSource != null}, Remote source: ${remoteSource != null}"
+                }
+
+                when {
+                    localSource != null && remoteSource == null -> {
+                        logger.debug { "Using local source: ${localSource.name}." }
+                        localSource
+                    }
+
+                    remoteSource != null && localSource == null -> {
+                        logger.debug { "Using remote source: ${remoteSource.name}." }
+                        remoteSource
+                    }
+
+                    else -> {
+                        logger.debug { "Remote and local have the same source ID: $sourceId. Keeping local." }
+                        localSource
+                    }
+                }
+            }
+
+        logger.debug { "Source merge completed. Total merged sources: ${mergedSources.size}" }
+
+        return mergedSources
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt

@@ -0,0 +1,28 @@
+package suwayomi.tachidesk.graphql.mutations
+
+import suwayomi.tachidesk.global.impl.sync.SyncManager
+import suwayomi.tachidesk.graphql.directives.RequireAuth
+import suwayomi.tachidesk.graphql.types.StartSyncResult
+
+class SyncMutation {
+    data class StartSyncInput(
+        val clientMutationId: String? = null,
+    )
+
+    data class StartSyncPayload(
+        val clientMutationId: String? = null,
+        val result: StartSyncResult,
+    )
+
+    @RequireAuth
+    fun startSync(input: StartSyncInput): StartSyncPayload {
+        val (clientMutationId) = input
+
+        val result = SyncManager.startSync()
+
+        return StartSyncPayload(
+            clientMutationId = clientMutationId,
+            result = result,
+        )
+    }
+}

server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SyncQuery.kt

@@ -0,0 +1,11 @@
+package suwayomi.tachidesk.graphql.queries
+
+import suwayomi.tachidesk.global.impl.sync.SyncManager
+import suwayomi.tachidesk.graphql.directives.RequireAuth
+import suwayomi.tachidesk.graphql.types.SyncStatus
+import suwayomi.tachidesk.graphql.types.toStatus
+
+class SyncQuery {
+    @RequireAuth
+    fun lastSyncStatus(): SyncStatus? = SyncManager.lastSyncState.value?.toStatus()
+}

server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt

@@ -27,6 +27,7 @@ import suwayomi.tachidesk.graphql.mutations.MangaMutation
 import suwayomi.tachidesk.graphql.mutations.MetaMutation
 import suwayomi.tachidesk.graphql.mutations.SettingsMutation
 import suwayomi.tachidesk.graphql.mutations.SourceMutation
+import suwayomi.tachidesk.graphql.mutations.SyncMutation
 import suwayomi.tachidesk.graphql.mutations.TrackMutation
 import suwayomi.tachidesk.graphql.mutations.UpdateMutation
 import suwayomi.tachidesk.graphql.mutations.UserMutation
@@ -41,6 +42,7 @@ import suwayomi.tachidesk.graphql.queries.MangaQuery
 import suwayomi.tachidesk.graphql.queries.MetaQuery
 import suwayomi.tachidesk.graphql.queries.SettingsQuery
 import suwayomi.tachidesk.graphql.queries.SourceQuery
+import suwayomi.tachidesk.graphql.queries.SyncQuery
 import suwayomi.tachidesk.graphql.queries.TrackQuery
 import suwayomi.tachidesk.graphql.queries.UpdateQuery
 import suwayomi.tachidesk.graphql.server.primitives.Cursor
@@ -50,6 +52,7 @@ import suwayomi.tachidesk.graphql.server.primitives.GraphQLLongAsString
 import suwayomi.tachidesk.graphql.server.primitives.GraphQLUpload
 import suwayomi.tachidesk.graphql.subscriptions.DownloadSubscription
 import suwayomi.tachidesk.graphql.subscriptions.InfoSubscription
+import suwayomi.tachidesk.graphql.subscriptions.SyncSubscription
 import suwayomi.tachidesk.graphql.subscriptions.UpdateSubscription
 import kotlin.reflect.KClass
 import kotlin.reflect.KType
@@ -98,6 +101,7 @@ val schema =
                 TopLevelObject(MetaQuery()),
                 TopLevelObject(SettingsQuery()),
                 TopLevelObject(SourceQuery()),
+                TopLevelObject(SyncQuery()),
                 TopLevelObject(TrackQuery()),
                 TopLevelObject(UpdateQuery()),
             ),
@@ -114,6 +118,7 @@ val schema =
                 TopLevelObject(MangaMutation()),
                 TopLevelObject(MetaMutation()),
                 TopLevelObject(SettingsMutation()),
+                TopLevelObject(SyncMutation()),
                 TopLevelObject(SourceMutation()),
                 TopLevelObject(TrackMutation()),
                 TopLevelObject(UpdateMutation()),
@@ -123,6 +128,7 @@ val schema =
             listOf(
                 TopLevelObject(DownloadSubscription()),
                 TopLevelObject(InfoSubscription()),
+                TopLevelObject(SyncSubscription()),
                 TopLevelObject(UpdateSubscription()),
             ),
     )

server/src/main/kotlin/suwayomi/tachidesk/graphql/subscriptions/SyncSubscription.kt

@@ -0,0 +1,17 @@
+package suwayomi.tachidesk.graphql.subscriptions
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.filterNotNull
+import kotlinx.coroutines.flow.map
+import suwayomi.tachidesk.global.impl.sync.SyncManager
+import suwayomi.tachidesk.graphql.directives.RequireAuth
+import suwayomi.tachidesk.graphql.types.SyncStatus
+import suwayomi.tachidesk.graphql.types.toStatus
+
+class SyncSubscription {
+    @RequireAuth
+    fun syncStatusChanged(): Flow<SyncStatus> =
+        SyncManager.lastSyncState
+            .filterNotNull()
+            .map { it.toStatus() }
+}

server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SyncType.kt

@@ -0,0 +1,91 @@
+package suwayomi.tachidesk.graphql.types
+
+import suwayomi.tachidesk.global.impl.sync.SyncManager
+
+enum class StartSyncResult {
+    SUCCESS,
+    SYNC_IN_PROGRESS,
+    SYNC_DISABLED,
+}
+
+enum class SyncState {
+    STARTED,
+    CREATING_BACKUP,
+    DOWNLOADING,
+    MERGING,
+    UPLOADING,
+    RESTORING,
+    SUCCESS,
+    ERROR,
+}
+
+data class SyncStatus(
+    val state: SyncState,
+    val startDate: Long,
+    val endDate: Long? = null,
+    val backupRestoreId: String? = null,
+    val errorMessage: String? = null,
+)
+
+fun SyncManager.SyncState.toStatus(): SyncStatus =
+    when (this) {
+        is SyncManager.SyncState.Started -> {
+            SyncStatus(
+                state = SyncState.STARTED,
+                startDate = startDate.toEpochMilliseconds(),
+            )
+        }
+
+        is SyncManager.SyncState.CreatingBackup -> {
+            SyncStatus(
+                state = SyncState.CREATING_BACKUP,
+                startDate = startDate.toEpochMilliseconds(),
+            )
+        }
+
+        is SyncManager.SyncState.Downloading -> {
+            SyncStatus(
+                state = SyncState.DOWNLOADING,
+                startDate = startDate.toEpochMilliseconds(),
+            )
+        }
+
+        is SyncManager.SyncState.Merging -> {
+            SyncStatus(
+                state = SyncState.MERGING,
+                startDate = startDate.toEpochMilliseconds(),
+            )
+        }
+
+        is SyncManager.SyncState.Uploading -> {
+            SyncStatus(
+                state = SyncState.UPLOADING,
+                startDate = startDate.toEpochMilliseconds(),
+            )
+        }
+
+        is SyncManager.SyncState.Restoring -> {
+            SyncStatus(
+                state = SyncState.RESTORING,
+                startDate = startDate.toEpochMilliseconds(),
+                backupRestoreId = restoreId,
+            )
+        }
+
+        is SyncManager.SyncState.Success -> {
+            SyncStatus(
+                state = SyncState.SUCCESS,
+                startDate = startDate.toEpochMilliseconds(),
+                endDate = endDate.toEpochMilliseconds(),
+            )
+        }
+
+        is SyncManager.SyncState.Error -> {
+            SyncStatus(
+                state = SyncState.ERROR,
+                startDate = startDate.toEpochMilliseconds(),
+                endDate = endDate.toEpochMilliseconds(),
+                errorMessage = message,
+            )
+        }
+    }

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt

@@ -85,6 +85,12 @@ object CategoryManga {
         }
     }
 
+    fun removeMangaFromAllCategories(mangaId: Int) {
+        transaction {
+            CategoryMangaTable.deleteWhere { CategoryMangaTable.manga eq mangaId }
+        }
+    }
+
     /**
      * list of mangas that belong to a category
      */

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt

@@ -129,6 +129,8 @@ object Chapter {
                 downloaded = dbChapter[ChapterTable.isDownloaded],
                 pageCount = dbChapter[ChapterTable.pageCount],
                 chapterCount = chapterList.size,
+                lastModifiedAt = dbChapter[ChapterTable.lastModifiedAt],
+                version = dbChapter[ChapterTable.version],
                 meta = chapterMetas.getValue(dbChapter[ChapterTable.id].value),
             )
         }
@@ -279,6 +281,8 @@ object Chapter {
                                 this[ChapterTable.isRead] = false
                                 this[ChapterTable.isBookmarked] = false
                                 this[ChapterTable.isDownloaded] = false
+                                this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt
+                                this[ChapterTable.version] = chapter.version
                                 this[ChapterTable.pageCount] = -1
 
                                 // is recognized chapter number
@@ -322,6 +326,8 @@ object Chapter {
                                     this[ChapterTable.scanlator] = it.scanlator
                                     this[ChapterTable.sourceOrder] = it.index
                                     this[ChapterTable.realUrl] = it.realUrl
+                                    this[ChapterTable.lastModifiedAt] = it.lastModifiedAt
+                                    this[ChapterTable.version] = it.version
                                     this[ChapterTable.isDownloaded] = currentChapter.downloaded
                                     this[ChapterTable.pageCount] = currentChapter.pageCount
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt

@@ -99,6 +99,8 @@ object Manga {
                 updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
                 freshData = true,
                 trackers = Track.getTrackRecordsByMangaId(mangaId),
+                lastModifiedAt = mangaEntry[MangaTable.lastModifiedAt],
+                version = mangaEntry[MangaTable.version],
             )
         }
     }
@@ -246,6 +248,8 @@ object Manga {
         updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]),
         freshData = false,
         trackers = Track.getTrackRecordsByMangaId(mangaId),
+        lastModifiedAt = mangaEntry[MangaTable.lastModifiedAt],
+        version = mangaEntry[MangaTable.version],
     )
 
     fun getMangaMetaMap(mangaId: Int): Map<String, String> =

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt

@@ -21,6 +21,9 @@ import kotlinx.coroutines.sync.withLock
 import okio.buffer
 import okio.gzip
 import okio.source
+import org.jetbrains.exposed.v1.core.eq
+import org.jetbrains.exposed.v1.jdbc.transactions.transaction
+import org.jetbrains.exposed.v1.jdbc.update
 import suwayomi.tachidesk.graphql.types.toStatus
 import suwayomi.tachidesk.manga.impl.backup.BackupFlags
 import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult
@@ -31,6 +34,8 @@ import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler
 import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup
+import suwayomi.tachidesk.manga.model.table.ChapterTable
+import suwayomi.tachidesk.manga.model.table.MangaTable
 import java.io.InputStream
 import java.util.Date
 import java.util.Timer
@@ -109,6 +114,7 @@ object ProtoBackupImport : ProtoBackupBase() {
     fun restore(
         sourceStream: InputStream,
         flags: BackupFlags,
+        isSync: Boolean = false,
     ): String {
         val restoreId = System.currentTimeMillis().toString()
 
@@ -117,7 +123,7 @@ object ProtoBackupImport : ProtoBackupBase() {
         updateRestoreState(restoreId, BackupRestoreState.Idle)
 
         GlobalScope.launch {
-            restoreLegacy(sourceStream, restoreId, flags)
+            restoreLegacy(sourceStream, restoreId, flags, isSync)
         }
 
         return restoreId
@@ -127,11 +133,12 @@ object ProtoBackupImport : ProtoBackupBase() {
         sourceStream: InputStream,
         restoreId: String = "legacy",
         flags: BackupFlags = BackupFlags.DEFAULT,
+        isSync: Boolean = false,
     ): ValidationResult =
         backupMutex.withLock {
             try {
                 logger.info { "restore($restoreId): restoring..." }
-                performRestore(restoreId, sourceStream, flags)
+                performRestore(restoreId, sourceStream, flags, isSync)
             } catch (e: Exception) {
                 logger.error(e) { "restore($restoreId): failed due to" }
 
@@ -152,12 +159,14 @@ object ProtoBackupImport : ProtoBackupBase() {
         id: String,
         sourceStream: InputStream,
         flags: BackupFlags,
+        isSync: Boolean,
     ): ValidationResult {
         val backupString =
             sourceStream
                 .source()
-                .gzip()
+                .run {
-                .buffer()
+                    if (!isSync) gzip() else this
+                }.buffer()
                 .use { it.readByteArray() }
         val backup = parser.decodeFromByteArray(Backup.serializer(), backupString)
 
@@ -235,6 +244,17 @@ object ProtoBackupImport : ProtoBackupBase() {
             """.trimIndent()
         }
 
+        if (isSync) {
+            transaction {
+                MangaTable.update({ MangaTable.isSyncing eq true }) {
+                    it[isSyncing] = false
+                }
+                ChapterTable.update({ ChapterTable.isSyncing eq true }) {
+                    it[isSyncing] = false
+                }
+            }
+        }
+
         updateRestoreState(id, BackupRestoreState.Success)
 
         return validationResult

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt

@@ -8,7 +8,11 @@ package suwayomi.tachidesk.manga.impl.backup.proto.handlers
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 import org.jetbrains.exposed.v1.core.SortOrder
+import org.jetbrains.exposed.v1.core.eq
+import org.jetbrains.exposed.v1.jdbc.insertAndGetId
 import org.jetbrains.exposed.v1.jdbc.selectAll
+import org.jetbrains.exposed.v1.jdbc.transactions.transaction
+import org.jetbrains.exposed.v1.jdbc.update
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas
 import suwayomi.tachidesk.manga.impl.backup.BackupFlags
@@ -38,6 +42,9 @@ object BackupCategoryHandler {
                     it.name,
                     it.order,
                     0, // not supported in Tachidesk
+                    it.version,
+                    it.uid,
+                    it.lastModifiedAt,
                 ).apply {
                     this.meta = categoryToMeta[it.id] ?: emptyMap()
                 }
@@ -45,7 +52,56 @@ object BackupCategoryHandler {
         }
 
     fun restore(backupCategories: List<BackupCategory>): Map<Int, Int> {
-        val categoryIds = Category.createCategories(backupCategories.map { it.name })
+        val dbCategories = Category.getCategoryList()
+        val dbCategoriesByName = dbCategories.associateBy { it.name }
+        val dbCategoriesByUid = dbCategories.associateBy { it.uid }
+
+        var nextOrder = dbCategories.maxOfOrNull { it.order }?.plus(1) ?: 0
+
+        val categoryIds =
+            transaction {
+                backupCategories
+                    .map { backupCategory ->
+                        var dbCategory =
+                            if (backupCategory.uid != 0L) {
+                                dbCategoriesByUid[backupCategory.uid]
+                            } else {
+                                null
+                            }
+
+                        if (dbCategory == null) {
+                            dbCategory = dbCategoriesByName[backupCategory.name]
+                        }
+
+                        if (dbCategory != null) {
+                            CategoryTable.update({ CategoryTable.id eq dbCategory.id }) {
+                                it[name] = backupCategory.name
+                                it[order] = backupCategory.order
+                                it[version] = backupCategory.version
+                                it[uid] = if (backupCategory.uid != 0L) backupCategory.uid else dbCategory.uid
+                                it[lastModifiedAt] = backupCategory.lastModifiedAt
+                                it[isSyncing] = true
+                            }
+                            return@map dbCategory.id
+                        }
+
+                        val currentOrder = nextOrder++
+                        CategoryTable
+                            .insertAndGetId {
+                                it[name] = backupCategory.name
+                                it[order] = currentOrder
+                                it[version] = backupCategory.version
+                                it[uid] = backupCategory.uid
+                                it[lastModifiedAt] = backupCategory.lastModifiedAt
+                            }.value
+                    }
+            }
+
+        transaction {
+            CategoryTable.update({ CategoryTable.isSyncing eq true }) {
+                it[isSyncing] = false
+            }
+        }
 
         val metaEntryByCategoryId =
             categoryIds

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt

@@ -75,6 +75,8 @@ object BackupMangaHandler {
                         dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds,
                         viewer = 0, // not supported in Tachidesk
                         updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]),
+                        lastModifiedAt = mangaRow[MangaTable.lastModifiedAt],
+                        version = mangaRow[MangaTable.version],
                     )
 
                 val mangaId = mangaRow[MangaTable.id].value
@@ -110,6 +112,8 @@ object BackupMangaHandler {
                                     it.uploadDate,
                                     it.chapterNumber,
                                     chapters.size - it.index,
+                                    it.lastModifiedAt,
+                                    it.version,
                                 ).apply {
                                     if (flags.includeClientData) {
                                         this.meta = chapterToMeta[it.id] ?: emptyMap()
@@ -232,6 +236,9 @@ object BackupMangaHandler {
                                 it[inLibrary] = manga.favorite
 
                                 it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
+
+                                it[lastModifiedAt] = manga.lastModifiedAt
+                                it[version] = manga.version
                             }.value
                     } else {
                         val dbMangaId = dbManga[MangaTable.id].value
@@ -251,6 +258,9 @@ object BackupMangaHandler {
                             it[inLibrary] = manga.favorite || dbManga[inLibrary]
 
                             it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds
+
+                            it[lastModifiedAt] = manga.lastModifiedAt
+                            it[version] = manga.version
                         }
 
                         dbMangaId
@@ -268,7 +278,7 @@ object BackupMangaHandler {
                     restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags)
                 }
 
-                // merge categories
+                // update categories
                 if (flags.includeCategories) {
                     restoreMangaCategoryData(mangaId, categoryIds)
                 }
@@ -339,6 +349,9 @@ object BackupMangaHandler {
                             this[ChapterTable.lastReadAt] =
                                 historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0
                         }
+
+                        this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt
+                        this[ChapterTable.version] = chapter.version
                     }.map { it[ChapterTable.id].value }
             } else {
                 emptyList()
@@ -387,6 +400,7 @@ object BackupMangaHandler {
         mangaId: Int,
         categoryIds: List<Int>,
     ) {
+        CategoryManga.removeMangaFromAllCategories(mangaId)
         CategoryManga.addMangaToCategories(mangaId, categoryIds)
     }
 

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupCategory.kt

@@ -10,6 +10,10 @@ class BackupCategory(
     // @ProtoNumber(3) val updateInterval: Int = 0, 1.x value not used in 0.x
     // Bump by 100 to specify this is a 0.x value
     @ProtoNumber(100) var flags: Int = 0,
+    // syncyomi
+    @ProtoNumber(601) var version: Long = 0,
+    @ProtoNumber(602) var uid: Long = 0,
+    @ProtoNumber(603) var lastModifiedAt: Long = 0,
     // suwayomi
     @ProtoNumber(9000) var meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt

@@ -19,6 +19,9 @@ data class BackupChapter(
     // chapterNumber is called number is 1.x
     @ProtoNumber(9) var chapterNumber: Float = 0F,
     @ProtoNumber(10) var sourceOrder: Int = 0,
+    // syncyomi
+    @ProtoNumber(11) var lastModifiedAt: Long = 0,
+    @ProtoNumber(12) var version: Long = 0,
     // suwayomi
     @ProtoNumber(9000) var meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt

@@ -34,6 +34,9 @@ data class BackupManga(
     @ProtoNumber(103) var viewer_flags: Int? = null,
     @ProtoNumber(104) var history: List<BackupHistory> = emptyList(),
     @ProtoNumber(105) var updateStrategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE,
+    // syncyomi
+    @ProtoNumber(106) var lastModifiedAt: Long = 0,
+    @ProtoNumber(109) var version: Long = 0,
     // suwayomi
     @ProtoNumber(9000) var meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

@@ -28,6 +28,7 @@ import kotlinx.coroutines.flow.update
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Semaphore
 import kotlinx.coroutines.sync.withPermit
+import suwayomi.tachidesk.global.impl.sync.SyncManager
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.Chapter
@@ -337,90 +338,98 @@ class Updater : IUpdater {
         clear: Boolean?,
         forceAll: Boolean,
     ) {
-        saveLastUpdateTimestamp()
-
-        if (clear == true) {
-            reset()
-        }
-
-        val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
-        val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.EXCLUDE].orEmpty()
-        val includedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.INCLUDE].orEmpty()
-        val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.UNSET].orEmpty()
-        val categoriesToUpdate =
-            if (forceAll) {
-                categories
-            } else {
-                includedCategories.ifEmpty { unsetCategories }
-            }
-        val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
-        val updateStatusCategories =
-            mapOf(
-                Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
-                Pair(CategoryUpdateStatus.SKIPPED, skippedCategories),
-            )
-
-        logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
-
-        val categoriesToUpdateMangas =
-            categoriesToUpdate
-                .flatMap { CategoryManga.getCategoryMangaList(it.id) }
-                .distinctBy { it.id }
-        val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
-        val mangasToUpdate =
-            categoriesToUpdateMangas
-                .asSequence()
-                .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
-                .filter {
-                    if (serverConfig.excludeUnreadChapters.value) {
-                        (it.unreadCount ?: 0L) == 0L
-                    } else {
-                        true
-                    }
-                }.filter {
-                    if (it.initialized && serverConfig.excludeNotStarted.value) {
-                        it.lastReadAt != null
-                    } else {
-                        true
-                    }
-                }.filter {
-                    if (serverConfig.excludeCompleted.value) {
-                        it.status != MangaStatus.COMPLETED.name
-                    } else {
-                        true
-                    }
-                }.filter { forceAll || !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
-                .toList()
-        val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
-
-        this.updateStatusCategories = updateStatusCategories
-        this.updateStatusSkippedMangas = skippedMangas
-
-        if (mangasToUpdate.isEmpty()) {
-            // In case no manga gets updated and no update job was running before, the client would never receive an info
-            // about its update request
-            scope.launch {
-                updateStatus(immediate = true)
-            }
-            return
-        }
-
         scope.launch {
-            updateStatus(
+            SyncManager.ensureSync()
-                categoryUpdates =
+
-                    updateStatusCategories[CategoryUpdateStatus.UPDATING]
+            saveLastUpdateTimestamp()
-                        ?.map {
+
-                            CategoryUpdateJob(it, CategoryUpdateStatus.UPDATING)
+            if (clear == true) {
-                        }.orEmpty(),
+                reset()
-                mangaUpdates = mangasToUpdate.map { UpdateJob(it) },
+            }
-                isRunning = true,
+
+            val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
+            val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.EXCLUDE].orEmpty()
+            val includedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.INCLUDE].orEmpty()
+            val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.UNSET].orEmpty()
+            val categoriesToUpdate =
+                if (forceAll) {
+                    categories
+                } else {
+                    includedCategories.ifEmpty { unsetCategories }
+                }
+            val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
+            val updateStatusCategories =
+                mapOf(
+                    Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
+                    Pair(CategoryUpdateStatus.SKIPPED, skippedCategories),
+                )
+
+            logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
+
+            val categoriesToUpdateMangas =
+                categoriesToUpdate
+                    .flatMap { CategoryManga.getCategoryMangaList(it.id) }
+                    .distinctBy { it.id }
+            val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
+            val mangasToUpdate =
+                categoriesToUpdateMangas
+                    .asSequence()
+                    .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
+                    .filter {
+                        if (serverConfig.excludeUnreadChapters.value) {
+                            (it.unreadCount ?: 0L) == 0L
+                        } else {
+                            true
+                        }
+                    }.filter {
+                        if (it.initialized && serverConfig.excludeNotStarted.value) {
+                            it.lastReadAt != null
+                        } else {
+                            true
+                        }
+                    }.filter {
+                        if (serverConfig.excludeCompleted.value) {
+                            it.status != MangaStatus.COMPLETED.name
+                        } else {
+                            true
+                        }
+                    }.filter {
+                        forceAll ||
+                            !excludedCategories.any { category ->
+                                mangasToCategoriesMap[it.id]?.contains(category) == true
+                            }
+                    }.toList()
+            val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
+
+            this@Updater.updateStatusCategories = updateStatusCategories
+            this@Updater.updateStatusSkippedMangas = skippedMangas
+
+            if (mangasToUpdate.isEmpty()) {
+                // In case no manga gets updated and no update job was running before, the client would never receive an info
+                // about its update request
+                scope.launch {
+                    updateStatus(immediate = true)
+                }
+                return@launch
+            }
+
+            scope.launch {
+                updateStatus(
+                    categoryUpdates =
+                        updateStatusCategories[CategoryUpdateStatus.UPDATING]
+                            ?.map {
+                                CategoryUpdateJob(it, CategoryUpdateStatus.UPDATING)
+                            }.orEmpty(),
+                    mangaUpdates = mangasToUpdate.map { UpdateJob(it) },
+                    isRunning = true,
+                )
+            }
+
+            addMangasToQueue(
+                mangasToUpdate
+                    .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)),
             )
         }
-
-        addMangasToQueue(
-            mangasToUpdate
-                .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)),
-        )
     }
 
     override fun addMangasToQueue(mangas: List<MangaDataClass>) {

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/CategoryDataClass.kt

@@ -30,5 +30,8 @@ data class CategoryDataClass(
     val size: Int,
     val includeInUpdate: IncludeOrExclude,
     val includeInDownload: IncludeOrExclude,
+    val version: Long,
+    val uid: Long,
+    val lastModifiedAt: Long,
     val meta: Map<String, String> = emptyMap(),
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt

@@ -38,6 +38,8 @@ data class ChapterDataClass(
     val pageCount: Int = -1,
     /** total chapter count, used to calculate if there's a next and prev chapter */
     val chapterCount: Int? = null,
+    val lastModifiedAt: Long = 0,
+    val version: Long = 0,
     /** used to store client specific values */
     val meta: Map<String, String> = emptyMap(),
 ) {

server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt

@@ -43,6 +43,8 @@ data class MangaDataClass(
     val age: Long? = if (lastFetchedAt == null) 0 else Instant.now().epochSecond.minus(lastFetchedAt),
     val chaptersAge: Long? = if (chaptersLastFetchedAt == null) null else Instant.now().epochSecond.minus(chaptersLastFetchedAt),
     val trackers: List<MangaTrackerDataClass>? = null,
+    val lastModifiedAt: Long = 0,
+    val version: Long = 0,
 ) {
     override fun toString(): String = "\"$title\" (id= $id) (sourceId= $sourceId)"
 }

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/CategoryTable.kt

@@ -19,6 +19,11 @@ object CategoryTable : IntIdTable() {
     val isDefault = bool("is_default").default(false)
     val includeInUpdate = integer("include_in_update").default(IncludeOrExclude.UNSET.value)
     val includeInDownload = integer("include_in_download").default(IncludeOrExclude.UNSET.value)
+
+    val version = long("version").default(0)
+    val uid = long("uid").default(0)
+    val lastModifiedAt = long("last_modified_at").default(0)
+    val isSyncing = bool("is_syncing").default(false)
 }
 
 fun CategoryTable.toDataClass(categoryEntry: ResultRow) =
@@ -30,5 +35,8 @@ fun CategoryTable.toDataClass(categoryEntry: ResultRow) =
         Category.getCategorySize(categoryEntry[id].value),
         IncludeOrExclude.fromValue(categoryEntry[includeInUpdate]),
         IncludeOrExclude.fromValue(categoryEntry[includeInDownload]),
+        categoryEntry[version],
+        categoryEntry[uid],
+        categoryEntry[lastModifiedAt],
         Category.getCategoryMetaMap(categoryEntry[id].value),
     )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt

@@ -42,6 +42,10 @@ object ChapterTable : IntIdTable() {
     val manga = reference("manga", MangaTable, ReferenceOption.CASCADE)
 
     val koreaderHash = varchar("koreader_hash", 32).nullable()
+
+    val lastModifiedAt = long("last_modified_at").default(0)
+    val version = long("version").default(0)
+    val isSyncing = bool("is_syncing").default(false)
 }
 
 fun ChapterTable.toDataClass(
@@ -83,4 +87,6 @@ fun ChapterTable.toDataClass(
         } else {
             emptyMap()
         },
+    lastModifiedAt = chapterEntry[lastModifiedAt],
+    version = chapterEntry[version],
 )

server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt

@@ -46,6 +46,10 @@ object MangaTable : IntIdTable() {
     val chaptersLastFetchedAt = long("chapters_last_fetched_at").default(0)
 
     val updateStrategy = varchar("update_strategy", 256).default(UpdateStrategy.ALWAYS_UPDATE.name)
+
+    val lastModifiedAt = long("last_modified_at").default(0)
+    val version = long("version").default(0)
+    val isSyncing = bool("is_syncing").default(false)
 }
 
 fun MangaTable.toDataClass(
@@ -76,6 +80,8 @@ fun MangaTable.toDataClass(
     lastFetchedAt = mangaEntry[lastFetchedAt],
     chaptersLastFetchedAt = mangaEntry[chaptersLastFetchedAt],
     updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy]),
+    lastModifiedAt = mangaEntry[lastModifiedAt],
+    version = mangaEntry[version],
 )
 
 enum class MangaStatus(

server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt

@@ -36,6 +36,7 @@ import org.koin.core.context.startKoin
 import org.koin.core.module.Module
 import org.koin.dsl.module
 import suwayomi.tachidesk.global.impl.KcefWebView.Companion.toCefCookie
+import suwayomi.tachidesk.global.impl.sync.SyncManager
 import suwayomi.tachidesk.graphql.types.DatabaseType
 import suwayomi.tachidesk.i18n.LocalizationHelper
 import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport
@@ -517,6 +518,8 @@ fun applicationSetup() {
     // start DownloadManager and restore + resume downloads
     DownloadManager.restoreAndResumeDownloads()
 
+    SyncManager.scheduleSyncTask()
+
     // asynchronously initialize CEF
     GlobalScope.launch {
         CEFManager.init()

server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0056_SyncYomi.kt

@@ -0,0 +1,224 @@
+package suwayomi.tachidesk.server.database.migration
+
+import de.neonew.exposed.migrations.helpers.SQLMigration
+import suwayomi.tachidesk.graphql.types.DatabaseType
+import suwayomi.tachidesk.server.serverConfig
+
+@Suppress("ClassName", "unused")
+class M0056_SyncYomi : SQLMigration() {
+    override val sql =
+        when (serverConfig.databaseType.value) {
+            DatabaseType.POSTGRESQL -> postgresQuery()
+            DatabaseType.H2 -> h2Query()
+        }
+
+    // language=postgresql
+    fun postgresQuery(): String =
+        """
+        ALTER TABLE manga ADD COLUMN version BIGINT NOT NULL DEFAULT 0;
+        ALTER TABLE manga ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE;
+        ALTER TABLE manga ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0;
+
+        ALTER TABLE chapter ADD COLUMN version BIGINT NOT NULL DEFAULT 0;
+        ALTER TABLE chapter ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE;
+        ALTER TABLE chapter ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0;
+
+        ALTER TABLE category ADD COLUMN version BIGINT NOT NULL DEFAULT 0;
+        ALTER TABLE category ADD COLUMN uid BIGINT NOT NULL DEFAULT 0;
+        ALTER TABLE category ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE;
+        ALTER TABLE category ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0;
+
+
+        CREATE OR REPLACE FUNCTION update_manga_version()
+        RETURNS trigger AS $$
+        BEGIN
+            IF NOT NEW.is_syncing
+               AND ROW(NEW.url, NEW.description, NEW.in_library)
+                   IS DISTINCT FROM
+                   ROW(OLD.url, OLD.description, OLD.in_library)
+            THEN
+                NEW.version := OLD.version + 1;
+            END IF;
+
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+
+        CREATE TRIGGER update_manga_version
+        AFTER UPDATE ON manga
+        FOR EACH ROW
+        EXECUTE FUNCTION update_manga_version();
+
+
+        CREATE OR REPLACE FUNCTION update_chapter_and_manga_version()
+        RETURNS trigger AS $$
+        BEGIN
+            IF NOT NEW.is_syncing
+               AND ROW(NEW.read, NEW.bookmark, NEW.last_page_read)
+                   IS DISTINCT FROM
+                   ROW(OLD.read, OLD.bookmark, OLD.last_page_read)
+            THEN
+                NEW.version := OLD.version + 1;
+        
+                UPDATE manga SET version = version + 1 WHERE id = NEW.manga AND is_syncing = FALSE;
+            END IF;
+
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+
+        CREATE TRIGGER update_chapter_and_manga_version
+        AFTER UPDATE ON chapter
+        FOR EACH ROW
+        EXECUTE FUNCTION update_chapter_and_manga_version();
+
+
+        CREATE OR REPLACE FUNCTION update_manga_last_modified_at()
+        RETURNS trigger AS $$
+        BEGIN
+            NEW.last_modified_at := EXTRACT(EPOCH FROM NOW());
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+        
+        CREATE TRIGGER update_manga_last_modified_at
+        BEFORE UPDATE OR INSERT ON manga
+        FOR EACH ROW
+        EXECUTE FUNCTION update_manga_last_modified_at();
+        
+        
+        CREATE OR REPLACE FUNCTION update_chapter_last_modified_at()
+        RETURNS trigger AS $$
+        BEGIN
+            NEW.last_modified_at := EXTRACT(EPOCH FROM NOW());
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+        
+        CREATE TRIGGER update_chapter_last_modified_at
+        BEFORE UPDATE OR INSERT ON chapter
+        FOR EACH ROW
+        EXECUTE FUNCTION update_chapter_last_modified_at();
+
+
+        CREATE OR REPLACE FUNCTION insert_manga_category_update_version()
+        RETURNS trigger AS $$
+        BEGIN
+            UPDATE manga SET version = version + 1 WHERE id = NEW.manga AND is_syncing = FALSE;
+
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+
+        CREATE TRIGGER insert_manga_category_update_version
+        AFTER INSERT ON categorymanga
+        FOR EACH ROW
+        EXECUTE FUNCTION insert_manga_category_update_version();
+        
+
+        CREATE OR REPLACE FUNCTION insert_category_uid()
+        RETURNS trigger AS $$
+        BEGIN
+            IF NEW.uid = 0 THEN
+                NEW.uid := RANDOM(1, 9223372036854775807);
+            END IF;
+
+            IF NEW.last_modified_at = 0 THEN
+                NEW.last_modified_at := EXTRACT(EPOCH FROM NOW());
+            END IF;
+
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+        
+        CREATE TRIGGER insert_category_uid
+        BEFORE INSERT ON category
+        FOR EACH ROW
+        EXECUTE FUNCTION insert_category_uid();
+        
+        
+        CREATE OR REPLACE FUNCTION update_category_version()
+        RETURNS trigger AS $$
+        BEGIN
+            IF NOT NEW.is_syncing
+               AND ROW(NEW.name, NEW.sort_order)
+                   IS DISTINCT FROM
+                   ROW(OLD.name, OLD.sort_order)
+            THEN
+                NEW.version := NEW.version + 1;
+                NEW.last_modified_at := EXTRACT(EPOCH FROM NOW());
+            END IF;
+
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+        
+        CREATE TRIGGER update_category_version
+        BEFORE UPDATE ON category
+        FOR EACH ROW
+        EXECUTE FUNCTION update_category_version();
+        """.trimIndent()
+
+    // language=h2
+    fun h2Query() =
+        """
+        ALTER TABLE manga ADD COLUMN version BIGINT NOT NULL DEFAULT 0;
+        ALTER TABLE manga ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE;
+        ALTER TABLE manga ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0;
+
+        ALTER TABLE chapter ADD COLUMN version BIGINT NOT NULL DEFAULT 0;
+        ALTER TABLE chapter ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE;
+        ALTER TABLE chapter ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0;
+
+        ALTER TABLE category ADD COLUMN version BIGINT NOT NULL DEFAULT 0;
+        ALTER TABLE category ADD COLUMN uid BIGINT NOT NULL DEFAULT 0;
+        ALTER TABLE category ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE;
+        ALTER TABLE category ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0;
+        
+
+        CREATE TRIGGER update_manga_version 
+        BEFORE UPDATE ON manga
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateMangaVersionTrigger";
+        
+        CREATE TRIGGER update_chapter_and_manga_version
+        BEFORE UPDATE ON chapter
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateChapterAndMangaVersionTrigger";
+        
+        CREATE TRIGGER update_manga_last_modified_at
+        BEFORE UPDATE ON manga
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateMangaLastModifiedAtTrigger";
+        
+        CREATE TRIGGER insert_manga_last_modified_at
+        BEFORE INSERT ON manga
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateMangaLastModifiedAtTrigger";
+        
+        CREATE TRIGGER update_chapter_last_modified_at
+        BEFORE UPDATE ON chapter
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateChapterLastModifiedAtTrigger";
+        
+        CREATE TRIGGER insert_chapter_last_modified_at
+        BEFORE INSERT ON chapter
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateChapterLastModifiedAtTrigger";
+        
+        CREATE TRIGGER insert_manga_category_update_version
+        AFTER INSERT ON categorymanga
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.InsertMangaCategoryUpdateVersionTrigger";
+        
+        CREATE TRIGGER insert_category_uid
+        BEFORE INSERT ON category
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.InsertCategoryUidTrigger";
+        
+        CREATE TRIGGER update_category_version
+        BEFORE UPDATE ON category
+        FOR EACH ROW
+        CALL "suwayomi.tachidesk.server.database.trigger.UpdateCategoryVersionTrigger";
+        """.trimIndent()
+}

server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt

@@ -0,0 +1,141 @@
+package suwayomi.tachidesk.server.database.trigger
+
+import org.h2.tools.TriggerAdapter
+import java.sql.Connection
+import java.sql.ResultSet
+import kotlin.random.Random
+import kotlin.time.Clock
+
+@Suppress("unused")
+class UpdateMangaVersionTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet,
+        newRow: ResultSet,
+    ) {
+        val isSyncing = newRow.getBoolean("is_syncing")
+        val hasChanged =
+            oldRow.getString("url") != newRow.getString("url") ||
+                oldRow.getString("description") != newRow.getString("description") ||
+                oldRow.getBoolean("in_library") != newRow.getBoolean("in_library")
+
+        if (!isSyncing && hasChanged) {
+            val currentVersion = newRow.getLong("version")
+            newRow.updateLong("version", currentVersion + 1)
+        }
+    }
+}
+
+@Suppress("unused")
+class UpdateChapterAndMangaVersionTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet,
+        newRow: ResultSet,
+    ) {
+        val isSyncing = newRow.getBoolean("is_syncing")
+        val hasChanged =
+            oldRow.getBoolean("read") != newRow.getBoolean("read") ||
+                oldRow.getBoolean("bookmark") != newRow.getBoolean("bookmark") ||
+                oldRow.getInt("last_page_read") != newRow.getInt("last_page_read")
+
+        if (!isSyncing && hasChanged) {
+            val currentVersion = newRow.getLong("version")
+            newRow.updateLong("version", currentVersion + 1)
+
+            val mangaId = newRow.getInt("manga")
+            conn
+                .prepareStatement(
+                    "UPDATE MANGA SET version = version + 1 WHERE id = ? AND NOT is_syncing",
+                ).use {
+                    it.setInt(1, mangaId)
+                    it.executeUpdate()
+                }
+        }
+    }
+}
+
+@Suppress("unused")
+class UpdateMangaLastModifiedAtTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet?,
+        newRow: ResultSet,
+    ) {
+        newRow.updateLong("last_modified_at", Clock.System.now().epochSeconds)
+    }
+}
+
+@Suppress("unused")
+class UpdateChapterLastModifiedAtTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet?,
+        newRow: ResultSet,
+    ) {
+        newRow.updateLong("last_modified_at", Clock.System.now().epochSeconds)
+    }
+}
+
+@Suppress("unused")
+class InsertMangaCategoryUpdateVersionTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet?,
+        newRow: ResultSet,
+    ) {
+        val mangaId = newRow.getInt("manga")
+
+        conn
+            .prepareStatement(
+                "UPDATE MANGA SET version = version + 1 WHERE id = ? AND NOT is_syncing",
+            ).use {
+                it.setInt(1, mangaId)
+                it.executeUpdate()
+            }
+    }
+}
+
+@Suppress("unused")
+class InsertCategoryUidTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet?,
+        newRow: ResultSet,
+    ) {
+        if (newRow.getLong("uid") == 0L) {
+            newRow.updateLong("uid", Random.nextLong(1, Long.MAX_VALUE))
+        }
+
+        if (newRow.getLong("last_modified_at") == 0L) {
+            newRow.updateLong(
+                "last_modified_at",
+                Clock.System.now().epochSeconds,
+            )
+        }
+    }
+}
+
+@Suppress("unused")
+class UpdateCategoryVersionTrigger : TriggerAdapter() {
+    override fun fire(
+        conn: Connection,
+        oldRow: ResultSet,
+        newRow: ResultSet,
+    ) {
+        val isSyncing = newRow.getBoolean("is_syncing")
+        val hasChanged =
+            oldRow.getString("name") != newRow.getString("name") ||
+                oldRow.getInt("sort_order") != newRow.getInt("sort_order")
+
+        if (!isSyncing && hasChanged) {
+            val currentVersion = newRow.getLong("version")
+            newRow.updateLong("version", currentVersion + 1)
+
+            newRow.updateLong(
+                "last_modified_at",
+                Clock.System.now().epochSeconds,
+            )
+        }
+    }
+}