From 811e15162b538338a11ce7085167244efcfff8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartu=20=C3=96zen?= Date: Fri, 5 Jun 2026 22:31:51 +0300 Subject: [PATCH] Implement SyncYomi (#1813) * Implement SyncYomi * Add ability to select what to sync * Properly fix default category bug * Add periodic sync * Add PostgreSQL support * Deschedule previous task * Check if SyncYomi is enabled in syncData function * Don't allow multiple syncs at the same time * Convert SyncYomiSyncService to object * Make startSync non-suspend * Return a result from startSync * Sync before library update * Improvements * Use NetworkHelper client * Lint * Use measureTime * Database improvements - Move entire sync operation into a single transaction - Stop loading all manga to memory * Revert "Database improvements" This reverts commit bee8d214c3a2c2d6ab342263f73b20fa6622f289. * Actual database improvements * Remove runBlocking * Remove title check * Update updateNonFavorites function * Update timeout code * Improve PostgreSQL query * Create lastSyncState variable * Create lastSyncStatus query * Convert lastSyncState to StateFlow * Create lastSyncStatusChange subscription * Replace backupRestoreStatus with backupRestoreId * Add startDate and endDate * Add logs for sync start and end * Handle all errors in syncData * Change category restore function to match Mihon's behavior * Fix comment * Remove duplicate BackupMangaHandler.backup call * Remove duplicated log * Rename subscription to syncStatusChanged * Use same flags for restoring * Update syncInterval config to use DurationSetting * Update sync scheduling logic * Reorder conditions to reduce database calls * Prevent deleted ghost chapters from reappearing during sync jobobby04/TachiyomiSY#1575 * Improve sync merging categories jobobby04/TachiyomiSY#1559 * Make columns not null * Improve H2 triggers * Add documentation --- docs/Configuring-Suwayomi‐Server.md | 22 + .../suwayomi/tachidesk/server/ServerConfig.kt | 61 ++ .../tachidesk/server/settings/SettingGroup.kt | 1 + .../tachidesk/global/impl/sync/SyncManager.kt | 519 ++++++++++++++++++ .../global/impl/sync/SyncYomiSyncService.kt | 517 +++++++++++++++++ .../graphql/mutations/SyncMutation.kt | 28 + .../tachidesk/graphql/queries/SyncQuery.kt | 11 + .../graphql/server/TachideskGraphQLSchema.kt | 6 + .../graphql/subscriptions/SyncSubscription.kt | 17 + .../tachidesk/graphql/types/SyncType.kt | 91 +++ .../tachidesk/manga/impl/CategoryManga.kt | 6 + .../suwayomi/tachidesk/manga/impl/Chapter.kt | 6 + .../suwayomi/tachidesk/manga/impl/Manga.kt | 4 + .../impl/backup/proto/ProtoBackupImport.kt | 28 +- .../proto/handlers/BackupCategoryHandler.kt | 58 +- .../proto/handlers/BackupMangaHandler.kt | 16 +- .../backup/proto/models/BackupCategory.kt | 4 + .../impl/backup/proto/models/BackupChapter.kt | 3 + .../impl/backup/proto/models/BackupManga.kt | 3 + .../tachidesk/manga/impl/update/Updater.kt | 171 +++--- .../model/dataclass/CategoryDataClass.kt | 3 + .../manga/model/dataclass/ChapterDataClass.kt | 2 + .../manga/model/dataclass/MangaDataClass.kt | 2 + .../manga/model/table/CategoryTable.kt | 8 + .../manga/model/table/ChapterTable.kt | 6 + .../tachidesk/manga/model/table/MangaTable.kt | 6 + .../suwayomi/tachidesk/server/ServerSetup.kt | 3 + .../database/migration/M0056_SyncYomi.kt | 224 ++++++++ .../database/trigger/SyncYomiTriggers.kt | 141 +++++ 29 files changed, 1880 insertions(+), 87 deletions(-) create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SyncQuery.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/graphql/subscriptions/SyncSubscription.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SyncType.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0056_SyncYomi.kt create mode 100644 server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt diff --git a/docs/Configuring-Suwayomi‐Server.md b/docs/Configuring-Suwayomi‐Server.md index 9a8787c4f..02c832177 100644 --- a/docs/Configuring-Suwayomi‐Server.md +++ b/docs/Configuring-Suwayomi‐Server.md @@ -276,6 +276,28 @@ server.useHikariConnectionPool = true - `server.databasePassword` the username with which to authenticate at the PostgreSQL instance. - `server.useHikariConnectionPool` use Hikari Connection Pool to connect to the database. +### SyncYomi +``` +server.syncYomiEnabled = false +server.syncYomiHost = "" +server.syncYomiApiKey = "" +server.syncDataManga = true +server.syncDataChapters = true +server.syncDataTracking = true +server.syncDataHistory = true +server.syncDataCategories = true +server.syncInterval = "0s" +``` +- `server.syncYomiEnabled` controls whether SyncYomi is enabled. +- `server.syncYomiHost` base URL of the SyncYomi server instance. e.g. `http://localhost:8282` +- `server.syncYomiApiKey` API key to authenticate with SyncYomi. You must use the same API key in both Suwayomi and SyncYomi. +- `server.syncDataManga` enables syncing manga. +- `server.syncDataChapters` enables syncing chapters. +- `server.syncDataTracking` enables syncing tracking data. +- `server.syncDataHistory` enables syncing reading history. +- `server.syncDataCategories` enables syncing categories. +- `server.syncInterval` interval between automatic sync operations. Use `0s` to disable. + **Note:** The example [docker-compose.yml file](https://github.com/Suwayomi/Suwayomi-Server-docker/blob/main/docker-compose.yml) contains everything you need to get started with Suwayomi+PostgreSQL. Please be aware that PostgreSQL support is currently still in beta. **Note:** These settings are excluded from backups, so a backup can be used to easily switch database installations by setting up the connection first, then restoring the backup. diff --git a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index 3e7acf37c..93b42e891 100644 --- a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -1038,7 +1038,68 @@ class ServerConfig( description = "Enable the WebView via CEF (Chromium)" ) + val syncYomiEnabled: MutableStateFlow by BooleanSetting( + protoNumber = 87, + defaultValue = false, + group = SettingGroup.SYNCYOMI, + privacySafe = true + ) + val syncYomiHost: MutableStateFlow by StringSetting( + protoNumber = 88, + defaultValue = "", + group = SettingGroup.SYNCYOMI, + privacySafe = true, + ) + + val syncYomiApiKey: MutableStateFlow by StringSetting( + protoNumber = 89, + defaultValue = "", + group = SettingGroup.SYNCYOMI, + privacySafe = false, + ) + + val syncDataManga: MutableStateFlow by BooleanSetting( + protoNumber = 90, + defaultValue = true, + group = SettingGroup.SYNCYOMI, + privacySafe = true, + ) + + val syncDataChapters: MutableStateFlow by BooleanSetting( + protoNumber = 91, + defaultValue = true, + group = SettingGroup.SYNCYOMI, + privacySafe = true, + ) + + val syncDataTracking: MutableStateFlow by BooleanSetting( + protoNumber = 92, + defaultValue = true, + group = SettingGroup.SYNCYOMI, + privacySafe = true, + ) + + val syncDataHistory: MutableStateFlow by BooleanSetting( + protoNumber = 93, + defaultValue = true, + group = SettingGroup.SYNCYOMI, + privacySafe = true, + ) + + val syncDataCategories: MutableStateFlow by BooleanSetting( + protoNumber = 94, + defaultValue = true, + group = SettingGroup.SYNCYOMI, + privacySafe = true, + ) + + val syncInterval: MutableStateFlow by DurationSetting( + protoNumber = 95, + defaultValue = 0.seconds, + group = SettingGroup.SYNCYOMI, + privacySafe = true, + ) /** ****************************************************************** **/ /** **/ diff --git a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt index 0acd9af88..eff184b3e 100644 --- a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt +++ b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt @@ -18,6 +18,7 @@ enum class SettingGroup( OPDS("OPDS"), KOREADER_SYNC("KOReader sync"), WEB_VIEW("WebView"), + SYNCYOMI("SyncYomi") ; override fun toString(): String = value diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt new file mode 100644 index 000000000..c26b535b9 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt @@ -0,0 +1,519 @@ +package suwayomi.tachidesk.global.impl.sync + +import android.app.Application +import android.content.Context +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.combine +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.serialization.Serializable +import kotlinx.serialization.protobuf.ProtoBuf +import org.jetbrains.exposed.v1.core.and +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update +import suwayomi.tachidesk.graphql.types.StartSyncResult +import suwayomi.tachidesk.manga.impl.Category +import suwayomi.tachidesk.manga.impl.Library.handleMangaThumbnail +import suwayomi.tachidesk.manga.impl.backup.BackupFlags +import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler +import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga +import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass +import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass +import suwayomi.tachidesk.manga.model.table.CategoryMangaTable +import suwayomi.tachidesk.manga.model.table.CategoryTable +import suwayomi.tachidesk.manga.model.table.ChapterTable +import suwayomi.tachidesk.manga.model.table.MangaTable +import suwayomi.tachidesk.manga.model.table.toDataClass +import suwayomi.tachidesk.server.serverConfig +import suwayomi.tachidesk.util.HAScheduler +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import kotlin.time.Clock +import kotlin.time.Duration.Companion.seconds +import kotlin.time.Instant +import kotlin.time.measureTime + +@Serializable +data class SyncData( + val backup: Backup? = null, +) + +object SyncManager { + private val syncPreferences = Injekt.get().getSharedPreferences("sync", Context.MODE_PRIVATE) + private val logger = KotlinLogging.logger {} + + private var currentTaskId: String? = null + private val syncMutex = Mutex() + + private val _lastSyncState: MutableStateFlow = MutableStateFlow(null) + val lastSyncState: StateFlow = _lastSyncState.asStateFlow() + + @OptIn(DelicateCoroutinesApi::class) + fun scheduleSyncTask() { + serverConfig.subscribeTo( + combine( + serverConfig.syncYomiEnabled, + serverConfig.syncInterval, + ) { enabled, interval -> Pair(enabled, interval) }, + { (enabled, interval) -> + currentTaskId?.let { HAScheduler.deschedule(it) } + + currentTaskId = + if (enabled && interval > 0.seconds) { + val lastSyncDate = + syncPreferences + .getLong("last_scheduled_sync", 0L) + .takeIf { it != 0L } + ?.let { Instant.fromEpochMilliseconds(it) } + + if (lastSyncDate == null) { + syncPreferences + .edit() + .putLong("last_scheduled_sync", Clock.System.now().toEpochMilliseconds()) + .apply() + } + + val delay = + if (lastSyncDate != null) { + ((interval) - (Clock.System.now() - lastSyncDate)).coerceAtLeast(0.seconds) + } else { + interval + } + + HAScheduler.schedule( + { + startSync(periodic = true) + + syncPreferences + .edit() + .putLong("last_scheduled_sync", Clock.System.now().toEpochMilliseconds()) + .apply() + }, + interval = interval.inWholeMilliseconds, + delay = delay.inWholeMilliseconds, + name = "sync", + ) + } else { + syncPreferences + .edit() + .remove("last_scheduled_sync") + .apply() + null + } + }, + ignoreInitialValue = false, + ) + } + + @OptIn(DelicateCoroutinesApi::class) + fun startSync(periodic: Boolean = false): StartSyncResult { + if (!serverConfig.syncYomiEnabled.value) { + return StartSyncResult.SYNC_DISABLED + } + + if (!syncMutex.tryLock()) { + return StartSyncResult.SYNC_IN_PROGRESS + } + + GlobalScope.launch { + try { + syncData(periodic) + } finally { + syncMutex.unlock() + } + } + + return StartSyncResult.SUCCESS + } + + suspend fun ensureSync() { + if (!serverConfig.syncYomiEnabled.value) { + return + } + + if (syncMutex.tryLock()) { + // there is no ongoing sync, so start one + try { + syncData() + } finally { + syncMutex.unlock() + } + } else { + // wait for the ongoing sync to finish + syncMutex.withLock {} + } + } + + private suspend fun syncData(periodic: Boolean = false) { + val startInstant = Clock.System.now() + _lastSyncState.value = SyncState.Started(startInstant) + + try { + logger.info { + if (periodic) { + "Starting periodic sync" + } else { + "Starting manual sync" + } + } + + transaction { + MangaTable.update({ MangaTable.isSyncing eq true }) { + it[isSyncing] = false + } + ChapterTable.update({ ChapterTable.isSyncing eq true }) { + it[isSyncing] = false + } + CategoryTable.update({ CategoryTable.isSyncing eq true }) { + it[isSyncing] = false + } + } + + val backupFlags = + BackupFlags( + includeManga = serverConfig.syncDataManga.value, + includeCategories = serverConfig.syncDataCategories.value, + includeChapters = serverConfig.syncDataChapters.value, + includeTracking = serverConfig.syncDataTracking.value, + includeHistory = serverConfig.syncDataHistory.value, + includeClientData = false, + includeServerSettings = false, + ) + + _lastSyncState.value = SyncState.CreatingBackup(startInstant) + val backupMangas = BackupMangaHandler.backup(backupFlags) + val backup = + Backup( + backupMangas, + BackupCategoryHandler.backup(backupFlags).filter { it.name != Category.DEFAULT_CATEGORY_NAME }, + BackupSourceHandler.backup(backupMangas, backupFlags), + emptyMap(), + null, + ) + + val syncData = + SyncData( + backup = backup, + ) + + val remoteBackup = + SyncYomiSyncService.doSync(syncData, startInstant) { + _lastSyncState.value = it + } + + if (remoteBackup == null) { + logger.debug { "Skip restore due to network issues" } + finishWithError(startInstant, "Network error", periodic) + return + } + + if (remoteBackup === syncData.backup) { + // nothing changed + logger.debug { "Skip restore due to remote was overwrite from local" } + finishWithSuccess(startInstant, periodic) + return + } + + // Stop the sync early if the remote backup is null or empty + if (remoteBackup.backupManga.isEmpty() && remoteBackup.backupCategories.isEmpty() && remoteBackup.backupSources.isEmpty()) { + logger.error { "No data found on remote server." } + finishWithError(startInstant, "No data found on remote server.", periodic) + return + } + + val isLibraryEmpty = + transaction { + MangaTable + .selectAll() + .where { MangaTable.inLibrary eq true } + .empty() + } + + // Check if it's first sync based on lastSyncTimestamp + if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && !isLibraryEmpty) { + // It's first sync no need to restore data. (just update remote data) + finishWithSuccess(startInstant, periodic) + return + } + + val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup) + updateNonFavorites(nonFavorites) + + val newSyncData = + backup.copy( + backupManga = filteredFavorites, + backupCategories = remoteBackup.backupCategories, + backupSources = remoteBackup.backupSources, + ) + + val hasMangaChanges = filteredFavorites.isNotEmpty() + val hasCategoryChanges = remoteBackup.backupCategories != backup.backupCategories + val hasSourceChanges = remoteBackup.backupSources != backup.backupSources + + if (!hasMangaChanges && !hasCategoryChanges && !hasSourceChanges) { + // update the sync timestamp + finishWithSuccess(startInstant, periodic) + return + } + + if (serverConfig.syncDataCategories.value) { + val mergedUids = newSyncData.backupCategories.map { it.uid }.toSet() + val mergedNames = newSyncData.backupCategories.map { it.name }.toSet() + val localCategories = Category.getCategoryList().filterNot { it.default } // Exclude system category + val categoriesToDelete = + localCategories.filter { + it.uid !in mergedUids && it.name !in mergedNames + } + if (categoriesToDelete.isNotEmpty()) { + transaction { + categoriesToDelete.forEach { + Category.removeCategory(it.id) + } + } + } + } + + val backupStream = ProtoBuf.encodeToByteArray(Backup.serializer(), newSyncData).inputStream() + val restoreId = + ProtoBackupImport.restore( + sourceStream = backupStream, + flags = backupFlags, + isSync = true, + ) + _lastSyncState.value = SyncState.Restoring(startInstant, restoreId) + + ProtoBackupImport.notifyFlow.first { + val restoreState = ProtoBackupImport.getRestoreState(restoreId) + + restoreState == ProtoBackupImport.BackupRestoreState.Success || + restoreState == ProtoBackupImport.BackupRestoreState.Failure + } + + // update the sync timestamp + finishWithSuccess(startInstant, periodic) + } catch (e: Throwable) { + logger.error { "Error syncing: ${e.message}" } + finishWithError(startInstant, "${e::class.qualifiedName}: ${e.message}", periodic) + } + } + + private fun finishWithSuccess( + startInstant: Instant, + periodic: Boolean, + ) { + syncPreferences + .edit() + .putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds()) + .apply() + _lastSyncState.value = SyncState.Success(startInstant) + + logger.info { + if (periodic) { + "Periodic sync completed successfully" + } else { + "Manual sync completed successfully" + } + } + } + + private fun finishWithError( + startInstant: Instant, + message: String, + periodic: Boolean, + ) { + _lastSyncState.value = SyncState.Error(startInstant, message) + + logger.info { + if (periodic) { + "Periodic sync failed: $message" + } else { + "Manual sync failed: $message" + } + } + } + + private fun isMangaDifferent( + localManga: MangaDataClass, + remoteManga: BackupManga, + ): Boolean { + if (localManga.version != remoteManga.version) { + return true + } + + val localChapters = + transaction { + ChapterTable + .selectAll() + .where { ChapterTable.manga eq localManga.id } + .map { ChapterTable.toDataClass(it) } + } + + if (areChaptersDifferent(localChapters, remoteManga.chapters)) { + return true + } + + val localCategories = + transaction { + CategoryMangaTable + .innerJoin(CategoryTable) + .selectAll() + .where { CategoryMangaTable.manga eq localManga.id } + .map { it[CategoryTable.order] } + } + + return localCategories.toSet() != remoteManga.categories.toSet() + } + + private fun areChaptersDifferent( + localChapters: List, + remoteChapters: List, + ): Boolean { + val localChapterMap = localChapters.associateBy { it.url } + val remoteChapterMap = remoteChapters.associateBy { it.url } + + if (localChapterMap.size != remoteChapterMap.size) { + return true + } + + for ((url, localChapter) in localChapterMap) { + val remoteChapter = remoteChapterMap[url] + + // If a matching remote chapter doesn't exist, or the version numbers are different, consider them different + if (remoteChapter == null || localChapter.version != remoteChapter.version) { + return true + } + } + + return false + } + + private fun filterFavoritesAndNonFavorites(backup: Backup): Pair, List> { + val favorites = mutableListOf() + val nonFavorites = mutableListOf() + + val elapsedTime = + measureTime { + logger.debug { "Starting to filter favorites and non-favorites from backup data." } + + backup.backupManga.forEach { remoteManga -> + val localManga = + transaction { + MangaTable + .selectAll() + .where { + (MangaTable.sourceReference eq remoteManga.source) and + (MangaTable.url eq remoteManga.url) + }.limit(1) + .map { MangaTable.toDataClass(it) } + .firstOrNull() + } + + when { + // Checks if the manga is in favorites and needs updating or adding + remoteManga.favorite -> { + if (localManga == null || isMangaDifferent(localManga, remoteManga)) { + logger.debug { "Adding to favorites: ${remoteManga.title}" } + favorites.add(remoteManga) + } else { + logger.debug { "Already up-to-date favorite: ${remoteManga.title}" } + } + } + + // Handle non-favorites + !remoteManga.favorite -> { + logger.debug { "Adding to non-favorites: ${remoteManga.title}" } + nonFavorites.add(remoteManga) + } + } + } + } + + logger.debug { + "Filtering completed in $elapsedTime. Favorites found: ${favorites.size}, Non-favorites found: ${nonFavorites.size}" + } + + return Pair(favorites, nonFavorites) + } + + private fun updateNonFavorites(nonFavorites: List) { + nonFavorites.forEach { nonFavorite -> + val localManga = + transaction { + MangaTable + .selectAll() + .where { + (MangaTable.sourceReference eq nonFavorite.source) and + (MangaTable.url eq nonFavorite.url) + }.limit(1) + .map { MangaTable.toDataClass(it) } + .firstOrNull() + } + + if (localManga != null) { + if (localManga.inLibrary != nonFavorite.favorite) { + transaction { + MangaTable.update({ MangaTable.id eq localManga.id }) { + it[inLibrary] = nonFavorite.favorite + } + }.apply { + handleMangaThumbnail(localManga.id, nonFavorite.favorite) + } + } + } + } + } + + sealed class SyncState( + open val startDate: Instant, + ) { + data class Started( + override val startDate: Instant, + ) : SyncState(startDate) + + data class CreatingBackup( + override val startDate: Instant, + ) : SyncState(startDate) + + data class Downloading( + override val startDate: Instant, + ) : SyncState(startDate) + + data class Merging( + override val startDate: Instant, + ) : SyncState(startDate) + + data class Uploading( + override val startDate: Instant, + ) : SyncState(startDate) + + data class Restoring( + override val startDate: Instant, + val restoreId: String, + ) : SyncState(startDate) + + data class Success( + override val startDate: Instant, + val endDate: Instant = Clock.System.now(), + ) : SyncState(startDate) + + data class Error( + override val startDate: Instant, + val message: String, + val endDate: Instant = Clock.System.now(), + ) : SyncState(startDate) + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt new file mode 100644 index 000000000..231c9bb73 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt @@ -0,0 +1,517 @@ +package suwayomi.tachidesk.global.impl.sync + +import android.app.Application +import android.content.Context +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.NetworkHelper +import eu.kanade.tachiyomi.network.PUT +import eu.kanade.tachiyomi.network.await +import io.github.oshai.kotlinlogging.KotlinLogging +import io.javalin.http.HttpStatus +import kotlinx.serialization.SerializationException +import kotlinx.serialization.protobuf.ProtoBuf +import okhttp3.Headers +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.RequestBody.Companion.toRequestBody +import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource +import suwayomi.tachidesk.server.serverConfig +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import uy.kohesive.injekt.injectLazy +import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds +import kotlin.time.Instant + +object SyncYomiSyncService { + private val syncPreferences = Injekt.get().getSharedPreferences("sync", Context.MODE_PRIVATE) + + private val network: NetworkHelper by injectLazy() + private val logger = KotlinLogging.logger {} + + private class SyncYomiException( + message: String?, + ) : Exception(message) + + suspend fun doSync( + syncData: SyncData, + startDate: Instant, + setSyncState: (SyncManager.SyncState) -> Unit, + ): Backup? { + setSyncState(SyncManager.SyncState.Downloading(startDate)) + val (remoteData, etag) = pullSyncData() + + val finalSyncData = + if (remoteData != null) { + require(etag.isNotEmpty()) { "ETag should never be empty if remote data is not null" } + logger.debug { "Try update remote data with ETag($etag)" } + setSyncState(SyncManager.SyncState.Merging(startDate)) + mergeSyncData(syncData, remoteData) + } else { + // init or overwrite remote data + logger.debug { "Try overwrite remote data with ETag($etag)" } + syncData + } + + if (finalSyncData.backup != null) { + setSyncState(SyncManager.SyncState.Uploading(startDate)) + } + pushSyncData(finalSyncData, etag) + return finalSyncData.backup + } + + private suspend fun pullSyncData(): Pair { + val host = serverConfig.syncYomiHost.value + val apiKey = serverConfig.syncYomiApiKey.value + val downloadUrl = "$host/api/sync/content" + + val headersBuilder = Headers.Builder().add("X-API-Token", apiKey) + val lastETag = syncPreferences.getString("last_sync_etag", "") ?: "" + if (lastETag != "") { + headersBuilder.add("If-None-Match", lastETag) + } + val headers = headersBuilder.build() + + val downloadRequest = + GET( + url = downloadUrl, + headers = headers, + ) + + val response = network.client.newCall(downloadRequest).await() + + if (response.code == HttpStatus.NOT_MODIFIED.code) { + // not modified + require(lastETag.isNotEmpty()) + logger.info { "Remote server not modified" } + return Pair(null, lastETag) + } else if (response.code == HttpStatus.NOT_FOUND.code) { + // maybe got deleted from remote + return Pair(null, "") + } + + if (response.isSuccessful) { + val newETag = + response.headers["ETag"] + ?.takeIf { it.isNotEmpty() } ?: throw SyncYomiException("Missing ETag") + + val byteArray = + response.body.byteStream().use { + return@use it.readBytes() + } + + return try { + val backup = ProtoBuf.decodeFromByteArray(Backup.serializer(), byteArray) + return Pair(SyncData(backup = backup), newETag) + } catch (_: SerializationException) { + logger.info { "Bad content responsed from server" } + // the body is invalid + // return default value so we can overwrite it + Pair(null, "") + } + } else { + val responseBody = response.body.string() + logger.error { "SyncError: $responseBody" } + throw SyncYomiException("Failed to download sync data: $responseBody") + } + } + + private suspend fun pushSyncData( + syncData: SyncData, + eTag: String, + ) { + val backup = syncData.backup ?: return + + val host = serverConfig.syncYomiHost.value + val apiKey = serverConfig.syncYomiApiKey.value + val uploadUrl = "$host/api/sync/content" + + val headersBuilder = Headers.Builder().add("X-API-Token", apiKey) + if (eTag.isNotEmpty()) { + headersBuilder.add("If-Match", eTag) + } + val headers = headersBuilder.build() + + // Set timeout to 30 seconds + val timeout = 30.seconds + val client = + network.client + .newBuilder() + .connectTimeout(timeout) + .readTimeout(timeout) + .writeTimeout(timeout) + .build() + + val byteArray = ProtoBuf.encodeToByteArray(Backup.serializer(), backup) + if (byteArray.isEmpty()) { + throw IllegalStateException("Empty backup error") + } + val body = byteArray.toRequestBody("application/octet-stream".toMediaType()) + + val uploadRequest = + PUT( + url = uploadUrl, + headers = headers, + body = body, + ) + + val response = client.newCall(uploadRequest).await() + + if (response.isSuccessful) { + val newETag = + response.headers["ETag"] + ?.takeIf { it.isNotEmpty() } ?: throw SyncYomiException("Missing ETag") + syncPreferences + .edit() + .putString("last_sync_etag", newETag) + .apply() + logger.debug { "SyncYomi sync completed" } + } else if (response.code == HttpStatus.PRECONDITION_FAILED.code) { + // other clients updated remote data, will try next time + logger.debug { "SyncYomi sync failed with 412" } + } else { + val responseBody = response.body.string() + logger.error { "SyncError: $responseBody" } + } + } + + fun mergeSyncData( + localSyncData: SyncData, + remoteSyncData: SyncData, + ): SyncData { + val mergedCategoriesList = + mergeCategoriesLists(localSyncData.backup?.backupCategories, remoteSyncData.backup?.backupCategories) + + val mergedMangaList = + mergeMangaLists( + localSyncData.backup?.backupManga, + remoteSyncData.backup?.backupManga, + localSyncData.backup?.backupCategories ?: emptyList(), + remoteSyncData.backup?.backupCategories ?: emptyList(), + mergedCategoriesList, + ) + + val mergedSourcesList = + mergeSourcesLists(localSyncData.backup?.backupSources, remoteSyncData.backup?.backupSources) + + // Create the merged Backup object + val mergedBackup = + Backup( + backupManga = mergedMangaList, + backupCategories = mergedCategoriesList, + backupSources = mergedSourcesList, + meta = emptyMap(), + serverSettings = null, + ) + + // Create the merged SData object + return SyncData( + backup = mergedBackup, + ) + } + + private fun mergeMangaLists( + localMangaList: List?, + remoteMangaList: List?, + localCategories: List, + remoteCategories: List, + mergedCategories: List, + ): List { + val localMangaListSafe = localMangaList.orEmpty() + val remoteMangaListSafe = remoteMangaList.orEmpty() + + logger.debug { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" } + + fun mangaCompositeKey(manga: BackupManga): String = + "${manga.source}|${manga.url}|${manga.title.lowercase().trim()}|${manga.author?.lowercase()?.trim()}" + + // Create maps using composite keys + val localMangaMap = localMangaListSafe.associateBy { mangaCompositeKey(it) } + val remoteMangaMap = remoteMangaListSafe.associateBy { mangaCompositeKey(it) } + + val localCategoriesMapByOrder = localCategories.associateBy { it.order } + val remoteCategoriesMapByOrder = remoteCategories.associateBy { it.order } + val mergedCategoriesMapByName = mergedCategories.associateBy { it.name } + + fun updateCategories( + theManga: BackupManga, + theMap: Map, + ): BackupManga = + theManga.copy( + categories = + theManga.categories.mapNotNull { + theMap[it]?.let { category -> + mergedCategoriesMapByName[category.name]?.order + } + }, + ) + + val lastSyncTime = syncPreferences.getLong("last_sync_timestamp", 0).milliseconds.inWholeSeconds + + val mergedList = + (localMangaMap.keys + remoteMangaMap.keys).distinct().mapNotNull { compositeKey -> + val local = localMangaMap[compositeKey] + val remote = remoteMangaMap[compositeKey] + + // New version comparison logic + when { + local != null && remote == null -> { + if (lastSyncTime == 0L || local.lastModifiedAt > lastSyncTime) { + updateCategories(local, localCategoriesMapByOrder) + } else { + logger.debug { "Dropping local manga deleted on remote: ${local.title}." } + null + } + } + + local == null && remote != null -> { + if (lastSyncTime == 0L || remote.lastModifiedAt > lastSyncTime) { + updateCategories(remote, remoteCategoriesMapByOrder) + } else { + logger.debug { "Dropping deleted remote manga: ${remote.title}." } + null + } + } + + local != null && remote != null -> { + // Compare versions to decide which manga to keep + if (local.version >= remote.version) { + logger.debug { "Keeping local version of ${local.title} with merged chapters." } + updateCategories( + local.copy( + chapters = + mergeChapters( + local.chapters, + remote.chapters, + lastSyncTime, + serverConfig.syncDataChapters.value, + ), + ), + localCategoriesMapByOrder, + ) + } else { + logger.debug { "Keeping remote version of ${remote.title} with merged chapters." } + updateCategories( + remote.copy( + chapters = + mergeChapters( + local.chapters, + remote.chapters, + lastSyncTime, + serverConfig.syncDataChapters.value, + ), + ), + remoteCategoriesMapByOrder, + ) + } + } + + else -> { + null + } // No manga found for key + } + } + + // Counting favorites and non-favorites + val (favorites, nonFavorites) = mergedList.partition { it.favorite } + + logger.debug { + "Merge completed. Total merged manga: ${mergedList.size}, Favorites: ${favorites.size}, Non-Favorites: ${nonFavorites.size}" + } + + return mergedList + } + + private fun mergeChapters( + localChapters: List, + remoteChapters: List, + lastSyncTime: Long, + syncingChapters: Boolean, + ): List { + if (!syncingChapters) { + return remoteChapters // If not syncing chapters, keep remote untouched + } + + fun chapterCompositeKey(chapter: BackupChapter): String = "${chapter.url}|${chapter.name}|${chapter.chapterNumber}" + + val localChapterMap = localChapters.associateBy { chapterCompositeKey(it) } + val remoteChapterMap = remoteChapters.associateBy { chapterCompositeKey(it) } + + logger.debug { "Starting chapter merge. Local chapters: ${localChapters.size}, Remote chapters: ${remoteChapters.size}" } + + // Merge both chapter maps based on version numbers + val mergedChapters = + (localChapterMap.keys + remoteChapterMap.keys).distinct().mapNotNull { compositeKey -> + val localChapter = localChapterMap[compositeKey] + val remoteChapter = remoteChapterMap[compositeKey] + + logger.debug { + "Processing chapter key: $compositeKey. Local chapter: ${localChapter != null}, Remote chapter: ${remoteChapter != null}" + } + + when { + localChapter != null && remoteChapter == null -> { + if (lastSyncTime == 0L || localChapter.lastModifiedAt > lastSyncTime) { + logger.debug { "Keeping local chapter: ${localChapter.name}." } + localChapter + } else { + logger.debug { "Dropping local chapter deleted on remote: ${localChapter.name}." } + null + } + } + + localChapter == null && remoteChapter != null -> { + if (lastSyncTime == 0L || remoteChapter.lastModifiedAt > lastSyncTime) { + logger.debug { "Taking remote chapter: ${remoteChapter.name}." } + remoteChapter + } else { + logger.debug { "Dropping deleted remote chapter: ${remoteChapter.name}." } + null + } + } + + localChapter != null && remoteChapter != null -> { + // Use version number to decide which chapter to keep + val chosenChapter = + if (localChapter.version >= remoteChapter.version) { + // If there mare more chapter on remote, local sourceOrder will need to be updated to maintain correct source order. + if (localChapters.size < remoteChapters.size) { + localChapter.copy(sourceOrder = remoteChapter.sourceOrder) + } else { + localChapter + } + } else { + remoteChapter + } + logger.debug { + "Merging chapter: ${chosenChapter.name}. Chosen version from: ${if (localChapter.version >= remoteChapter.version) "Local" else "Remote"}, Local version: ${localChapter.version}, Remote version: ${remoteChapter.version}." + } + chosenChapter + } + + else -> { + logger.debug { "No chapter found for composite key: $compositeKey. Skipping." } + null + } + } + } + + logger.debug { "Chapter merge completed. Total merged chapters: ${mergedChapters.size}" } + + return mergedChapters + } + + private fun mergeCategoriesLists( + localCategoriesList: List?, + remoteCategoriesList: List?, + ): List { + if (localCategoriesList == null) return remoteCategoriesList ?: emptyList() + if (remoteCategoriesList == null) return localCategoriesList + val result = mutableListOf() + val processedLocals = mutableSetOf() + + val localMapByUid = localCategoriesList.filter { it.uid != 0L }.associateBy { it.uid } + val localMapByName = localCategoriesList.associateBy { it.name } + + val lastSyncTime = syncPreferences.getLong("last_sync_timestamp", 0) + + remoteCategoriesList.forEach { remote -> + var localMatch: BackupCategory? = null + + // 1. Try match by UID + if (remote.uid != 0L) { + localMatch = localMapByUid[remote.uid] + } + + // 2. Try match by Name (fallback) + if (localMatch == null) { + localMatch = localMapByName[remote.name] + } + + if (localMatch != null) { + processedLocals.add(localMatch) + // Conflict resolution + if (localMatch.version >= remote.version) { + logger.debug { "Keeping local category: ${localMatch.name} (UID: ${localMatch.uid})" } + result.add(localMatch) + } else { + logger.debug { "Keeping remote category: ${remote.name} (UID: ${remote.uid})" } + // Preserve Local UID if Remote was 0 + if (remote.uid == 0L) { + remote.uid = localMatch.uid + } + result.add(remote) + } + } else { + val remoteModifiedTimeMillis = remote.lastModifiedAt.seconds.inWholeMilliseconds + if (lastSyncTime == 0L || remoteModifiedTimeMillis > lastSyncTime) { + logger.debug { "Adding new remote category: ${remote.name} (UID: ${remote.uid})" } + result.add(remote) + } else { + logger.debug { "Dropping deleted remote category: ${remote.name} (UID: ${remote.uid})" } + } + } + } + + // Add remaining Local Categories + localCategoriesList.forEach { local -> + if (local !in processedLocals) { + val localModifiedTimeMillis = local.lastModifiedAt.seconds.inWholeMilliseconds + if (lastSyncTime == 0L || localModifiedTimeMillis > lastSyncTime) { + logger.debug { "Keeping local only category: ${local.name} (UID: ${local.uid})" } + result.add(local) + } else { + logger.debug { "Dropping local category deleted on remote: ${local.name} (UID: ${local.uid})" } + } + } + } + + return result.sortedBy { it.order } + } + + private fun mergeSourcesLists( + localSources: List?, + remoteSources: List?, + ): List { + // Create maps using sourceId as key + val localSourceMap = localSources?.associateBy { it.sourceId } ?: emptyMap() + val remoteSourceMap = remoteSources?.associateBy { it.sourceId } ?: emptyMap() + + logger.debug { "Starting source merge. Local sources: ${localSources?.size}, Remote sources: ${remoteSources?.size}" } + + // Merge both source maps + val mergedSources = + (localSourceMap.keys + remoteSourceMap.keys).distinct().mapNotNull { sourceId -> + val localSource = localSourceMap[sourceId] + val remoteSource = remoteSourceMap[sourceId] + + logger.debug { + "Processing source ID: $sourceId. Local source: ${localSource != null}, Remote source: ${remoteSource != null}" + } + + when { + localSource != null && remoteSource == null -> { + logger.debug { "Using local source: ${localSource.name}." } + localSource + } + + remoteSource != null && localSource == null -> { + logger.debug { "Using remote source: ${remoteSource.name}." } + remoteSource + } + + else -> { + logger.debug { "Remote and local have the same source ID: $sourceId. Keeping local." } + localSource + } + } + } + + logger.debug { "Source merge completed. Total merged sources: ${mergedSources.size}" } + + return mergedSources + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt new file mode 100644 index 000000000..17b51a488 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt @@ -0,0 +1,28 @@ +package suwayomi.tachidesk.graphql.mutations + +import suwayomi.tachidesk.global.impl.sync.SyncManager +import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.types.StartSyncResult + +class SyncMutation { + data class StartSyncInput( + val clientMutationId: String? = null, + ) + + data class StartSyncPayload( + val clientMutationId: String? = null, + val result: StartSyncResult, + ) + + @RequireAuth + fun startSync(input: StartSyncInput): StartSyncPayload { + val (clientMutationId) = input + + val result = SyncManager.startSync() + + return StartSyncPayload( + clientMutationId = clientMutationId, + result = result, + ) + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SyncQuery.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SyncQuery.kt new file mode 100644 index 000000000..a2e820d70 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/SyncQuery.kt @@ -0,0 +1,11 @@ +package suwayomi.tachidesk.graphql.queries + +import suwayomi.tachidesk.global.impl.sync.SyncManager +import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.types.SyncStatus +import suwayomi.tachidesk.graphql.types.toStatus + +class SyncQuery { + @RequireAuth + fun lastSyncStatus(): SyncStatus? = SyncManager.lastSyncState.value?.toStatus() +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt index dd51ee552..d6527ae3a 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt @@ -27,6 +27,7 @@ import suwayomi.tachidesk.graphql.mutations.MangaMutation import suwayomi.tachidesk.graphql.mutations.MetaMutation import suwayomi.tachidesk.graphql.mutations.SettingsMutation import suwayomi.tachidesk.graphql.mutations.SourceMutation +import suwayomi.tachidesk.graphql.mutations.SyncMutation import suwayomi.tachidesk.graphql.mutations.TrackMutation import suwayomi.tachidesk.graphql.mutations.UpdateMutation import suwayomi.tachidesk.graphql.mutations.UserMutation @@ -41,6 +42,7 @@ import suwayomi.tachidesk.graphql.queries.MangaQuery import suwayomi.tachidesk.graphql.queries.MetaQuery import suwayomi.tachidesk.graphql.queries.SettingsQuery import suwayomi.tachidesk.graphql.queries.SourceQuery +import suwayomi.tachidesk.graphql.queries.SyncQuery import suwayomi.tachidesk.graphql.queries.TrackQuery import suwayomi.tachidesk.graphql.queries.UpdateQuery import suwayomi.tachidesk.graphql.server.primitives.Cursor @@ -50,6 +52,7 @@ import suwayomi.tachidesk.graphql.server.primitives.GraphQLLongAsString import suwayomi.tachidesk.graphql.server.primitives.GraphQLUpload import suwayomi.tachidesk.graphql.subscriptions.DownloadSubscription import suwayomi.tachidesk.graphql.subscriptions.InfoSubscription +import suwayomi.tachidesk.graphql.subscriptions.SyncSubscription import suwayomi.tachidesk.graphql.subscriptions.UpdateSubscription import kotlin.reflect.KClass import kotlin.reflect.KType @@ -98,6 +101,7 @@ val schema = TopLevelObject(MetaQuery()), TopLevelObject(SettingsQuery()), TopLevelObject(SourceQuery()), + TopLevelObject(SyncQuery()), TopLevelObject(TrackQuery()), TopLevelObject(UpdateQuery()), ), @@ -114,6 +118,7 @@ val schema = TopLevelObject(MangaMutation()), TopLevelObject(MetaMutation()), TopLevelObject(SettingsMutation()), + TopLevelObject(SyncMutation()), TopLevelObject(SourceMutation()), TopLevelObject(TrackMutation()), TopLevelObject(UpdateMutation()), @@ -123,6 +128,7 @@ val schema = listOf( TopLevelObject(DownloadSubscription()), TopLevelObject(InfoSubscription()), + TopLevelObject(SyncSubscription()), TopLevelObject(UpdateSubscription()), ), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/subscriptions/SyncSubscription.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/subscriptions/SyncSubscription.kt new file mode 100644 index 000000000..6bba2c2c4 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/subscriptions/SyncSubscription.kt @@ -0,0 +1,17 @@ +package suwayomi.tachidesk.graphql.subscriptions + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.filterNotNull +import kotlinx.coroutines.flow.map +import suwayomi.tachidesk.global.impl.sync.SyncManager +import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.graphql.types.SyncStatus +import suwayomi.tachidesk.graphql.types.toStatus + +class SyncSubscription { + @RequireAuth + fun syncStatusChanged(): Flow = + SyncManager.lastSyncState + .filterNotNull() + .map { it.toStatus() } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SyncType.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SyncType.kt new file mode 100644 index 000000000..5036664b3 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SyncType.kt @@ -0,0 +1,91 @@ +package suwayomi.tachidesk.graphql.types + +import suwayomi.tachidesk.global.impl.sync.SyncManager + +enum class StartSyncResult { + SUCCESS, + SYNC_IN_PROGRESS, + SYNC_DISABLED, +} + +enum class SyncState { + STARTED, + CREATING_BACKUP, + DOWNLOADING, + MERGING, + UPLOADING, + RESTORING, + SUCCESS, + ERROR, +} + +data class SyncStatus( + val state: SyncState, + val startDate: Long, + val endDate: Long? = null, + val backupRestoreId: String? = null, + val errorMessage: String? = null, +) + +fun SyncManager.SyncState.toStatus(): SyncStatus = + when (this) { + is SyncManager.SyncState.Started -> { + SyncStatus( + state = SyncState.STARTED, + startDate = startDate.toEpochMilliseconds(), + ) + } + + is SyncManager.SyncState.CreatingBackup -> { + SyncStatus( + state = SyncState.CREATING_BACKUP, + startDate = startDate.toEpochMilliseconds(), + ) + } + + is SyncManager.SyncState.Downloading -> { + SyncStatus( + state = SyncState.DOWNLOADING, + startDate = startDate.toEpochMilliseconds(), + ) + } + + is SyncManager.SyncState.Merging -> { + SyncStatus( + state = SyncState.MERGING, + startDate = startDate.toEpochMilliseconds(), + ) + } + + is SyncManager.SyncState.Uploading -> { + SyncStatus( + state = SyncState.UPLOADING, + startDate = startDate.toEpochMilliseconds(), + ) + } + + is SyncManager.SyncState.Restoring -> { + SyncStatus( + state = SyncState.RESTORING, + startDate = startDate.toEpochMilliseconds(), + backupRestoreId = restoreId, + ) + } + + is SyncManager.SyncState.Success -> { + SyncStatus( + state = SyncState.SUCCESS, + startDate = startDate.toEpochMilliseconds(), + endDate = endDate.toEpochMilliseconds(), + ) + } + + is SyncManager.SyncState.Error -> { + SyncStatus( + state = SyncState.ERROR, + startDate = startDate.toEpochMilliseconds(), + endDate = endDate.toEpochMilliseconds(), + errorMessage = message, + ) + } + } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt index 515858fb9..5aad83106 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/CategoryManga.kt @@ -85,6 +85,12 @@ object CategoryManga { } } + fun removeMangaFromAllCategories(mangaId: Int) { + transaction { + CategoryMangaTable.deleteWhere { CategoryMangaTable.manga eq mangaId } + } + } + /** * list of mangas that belong to a category */ diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index 5d0ef590a..641e386ab 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -129,6 +129,8 @@ object Chapter { downloaded = dbChapter[ChapterTable.isDownloaded], pageCount = dbChapter[ChapterTable.pageCount], chapterCount = chapterList.size, + lastModifiedAt = dbChapter[ChapterTable.lastModifiedAt], + version = dbChapter[ChapterTable.version], meta = chapterMetas.getValue(dbChapter[ChapterTable.id].value), ) } @@ -279,6 +281,8 @@ object Chapter { this[ChapterTable.isRead] = false this[ChapterTable.isBookmarked] = false this[ChapterTable.isDownloaded] = false + this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt + this[ChapterTable.version] = chapter.version this[ChapterTable.pageCount] = -1 // is recognized chapter number @@ -322,6 +326,8 @@ object Chapter { this[ChapterTable.scanlator] = it.scanlator this[ChapterTable.sourceOrder] = it.index this[ChapterTable.realUrl] = it.realUrl + this[ChapterTable.lastModifiedAt] = it.lastModifiedAt + this[ChapterTable.version] = it.version this[ChapterTable.isDownloaded] = currentChapter.downloaded this[ChapterTable.pageCount] = currentChapter.pageCount diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index 9ccf91930..0c5cec3c2 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -99,6 +99,8 @@ object Manga { updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]), freshData = true, trackers = Track.getTrackRecordsByMangaId(mangaId), + lastModifiedAt = mangaEntry[MangaTable.lastModifiedAt], + version = mangaEntry[MangaTable.version], ) } } @@ -246,6 +248,8 @@ object Manga { updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]), freshData = false, trackers = Track.getTrackRecordsByMangaId(mangaId), + lastModifiedAt = mangaEntry[MangaTable.lastModifiedAt], + version = mangaEntry[MangaTable.version], ) fun getMangaMetaMap(mangaId: Int): Map = diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt index a4f31f742..36c38bc25 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt @@ -21,6 +21,9 @@ import kotlinx.coroutines.sync.withLock import okio.buffer import okio.gzip import okio.source +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update import suwayomi.tachidesk.graphql.types.toStatus import suwayomi.tachidesk.manga.impl.backup.BackupFlags import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult @@ -31,6 +34,8 @@ import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup +import suwayomi.tachidesk.manga.model.table.ChapterTable +import suwayomi.tachidesk.manga.model.table.MangaTable import java.io.InputStream import java.util.Date import java.util.Timer @@ -109,6 +114,7 @@ object ProtoBackupImport : ProtoBackupBase() { fun restore( sourceStream: InputStream, flags: BackupFlags, + isSync: Boolean = false, ): String { val restoreId = System.currentTimeMillis().toString() @@ -117,7 +123,7 @@ object ProtoBackupImport : ProtoBackupBase() { updateRestoreState(restoreId, BackupRestoreState.Idle) GlobalScope.launch { - restoreLegacy(sourceStream, restoreId, flags) + restoreLegacy(sourceStream, restoreId, flags, isSync) } return restoreId @@ -127,11 +133,12 @@ object ProtoBackupImport : ProtoBackupBase() { sourceStream: InputStream, restoreId: String = "legacy", flags: BackupFlags = BackupFlags.DEFAULT, + isSync: Boolean = false, ): ValidationResult = backupMutex.withLock { try { logger.info { "restore($restoreId): restoring..." } - performRestore(restoreId, sourceStream, flags) + performRestore(restoreId, sourceStream, flags, isSync) } catch (e: Exception) { logger.error(e) { "restore($restoreId): failed due to" } @@ -152,12 +159,14 @@ object ProtoBackupImport : ProtoBackupBase() { id: String, sourceStream: InputStream, flags: BackupFlags, + isSync: Boolean, ): ValidationResult { val backupString = sourceStream .source() - .gzip() - .buffer() + .run { + if (!isSync) gzip() else this + }.buffer() .use { it.readByteArray() } val backup = parser.decodeFromByteArray(Backup.serializer(), backupString) @@ -235,6 +244,17 @@ object ProtoBackupImport : ProtoBackupBase() { """.trimIndent() } + if (isSync) { + transaction { + MangaTable.update({ MangaTable.isSyncing eq true }) { + it[isSyncing] = false + } + ChapterTable.update({ ChapterTable.isSyncing eq true }) { + it[isSyncing] = false + } + } + } + updateRestoreState(id, BackupRestoreState.Success) return validationResult diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt index 60f2cc1cb..fbad70fd7 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupCategoryHandler.kt @@ -8,7 +8,11 @@ package suwayomi.tachidesk.manga.impl.backup.proto.handlers * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ import org.jetbrains.exposed.v1.core.SortOrder +import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.jdbc.insertAndGetId import org.jetbrains.exposed.v1.jdbc.selectAll +import org.jetbrains.exposed.v1.jdbc.transactions.transaction +import org.jetbrains.exposed.v1.jdbc.update import suwayomi.tachidesk.manga.impl.Category import suwayomi.tachidesk.manga.impl.Category.modifyCategoriesMetas import suwayomi.tachidesk.manga.impl.backup.BackupFlags @@ -38,6 +42,9 @@ object BackupCategoryHandler { it.name, it.order, 0, // not supported in Tachidesk + it.version, + it.uid, + it.lastModifiedAt, ).apply { this.meta = categoryToMeta[it.id] ?: emptyMap() } @@ -45,7 +52,56 @@ object BackupCategoryHandler { } fun restore(backupCategories: List): Map { - val categoryIds = Category.createCategories(backupCategories.map { it.name }) + val dbCategories = Category.getCategoryList() + val dbCategoriesByName = dbCategories.associateBy { it.name } + val dbCategoriesByUid = dbCategories.associateBy { it.uid } + + var nextOrder = dbCategories.maxOfOrNull { it.order }?.plus(1) ?: 0 + + val categoryIds = + transaction { + backupCategories + .map { backupCategory -> + var dbCategory = + if (backupCategory.uid != 0L) { + dbCategoriesByUid[backupCategory.uid] + } else { + null + } + + if (dbCategory == null) { + dbCategory = dbCategoriesByName[backupCategory.name] + } + + if (dbCategory != null) { + CategoryTable.update({ CategoryTable.id eq dbCategory.id }) { + it[name] = backupCategory.name + it[order] = backupCategory.order + it[version] = backupCategory.version + it[uid] = if (backupCategory.uid != 0L) backupCategory.uid else dbCategory.uid + it[lastModifiedAt] = backupCategory.lastModifiedAt + it[isSyncing] = true + } + return@map dbCategory.id + } + + val currentOrder = nextOrder++ + CategoryTable + .insertAndGetId { + it[name] = backupCategory.name + it[order] = currentOrder + it[version] = backupCategory.version + it[uid] = backupCategory.uid + it[lastModifiedAt] = backupCategory.lastModifiedAt + }.value + } + } + + transaction { + CategoryTable.update({ CategoryTable.isSyncing eq true }) { + it[isSyncing] = false + } + } val metaEntryByCategoryId = categoryIds diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt index 9fb0d8e2f..bcd0cad04 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt @@ -75,6 +75,8 @@ object BackupMangaHandler { dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds, viewer = 0, // not supported in Tachidesk updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]), + lastModifiedAt = mangaRow[MangaTable.lastModifiedAt], + version = mangaRow[MangaTable.version], ) val mangaId = mangaRow[MangaTable.id].value @@ -110,6 +112,8 @@ object BackupMangaHandler { it.uploadDate, it.chapterNumber, chapters.size - it.index, + it.lastModifiedAt, + it.version, ).apply { if (flags.includeClientData) { this.meta = chapterToMeta[it.id] ?: emptyMap() @@ -232,6 +236,9 @@ object BackupMangaHandler { it[inLibrary] = manga.favorite it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds + + it[lastModifiedAt] = manga.lastModifiedAt + it[version] = manga.version }.value } else { val dbMangaId = dbManga[MangaTable.id].value @@ -251,6 +258,9 @@ object BackupMangaHandler { it[inLibrary] = manga.favorite || dbManga[inLibrary] it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds + + it[lastModifiedAt] = manga.lastModifiedAt + it[version] = manga.version } dbMangaId @@ -268,7 +278,7 @@ object BackupMangaHandler { restoreMangaChapterData(mangaId, restoreMode, chapters, history, flags) } - // merge categories + // update categories if (flags.includeCategories) { restoreMangaCategoryData(mangaId, categoryIds) } @@ -339,6 +349,9 @@ object BackupMangaHandler { this[ChapterTable.lastReadAt] = historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0 } + + this[ChapterTable.lastModifiedAt] = chapter.lastModifiedAt + this[ChapterTable.version] = chapter.version }.map { it[ChapterTable.id].value } } else { emptyList() @@ -387,6 +400,7 @@ object BackupMangaHandler { mangaId: Int, categoryIds: List, ) { + CategoryManga.removeMangaFromAllCategories(mangaId) CategoryManga.addMangaToCategories(mangaId, categoryIds) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupCategory.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupCategory.kt index 83a5c2b68..8b2bad80c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupCategory.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupCategory.kt @@ -10,6 +10,10 @@ class BackupCategory( // @ProtoNumber(3) val updateInterval: Int = 0, 1.x value not used in 0.x // Bump by 100 to specify this is a 0.x value @ProtoNumber(100) var flags: Int = 0, + // syncyomi + @ProtoNumber(601) var version: Long = 0, + @ProtoNumber(602) var uid: Long = 0, + @ProtoNumber(603) var lastModifiedAt: Long = 0, // suwayomi @ProtoNumber(9000) var meta: Map = emptyMap(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt index 145184226..cf3941453 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt @@ -19,6 +19,9 @@ data class BackupChapter( // chapterNumber is called number is 1.x @ProtoNumber(9) var chapterNumber: Float = 0F, @ProtoNumber(10) var sourceOrder: Int = 0, + // syncyomi + @ProtoNumber(11) var lastModifiedAt: Long = 0, + @ProtoNumber(12) var version: Long = 0, // suwayomi @ProtoNumber(9000) var meta: Map = emptyMap(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt index c295c70c7..3dc29f640 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt @@ -34,6 +34,9 @@ data class BackupManga( @ProtoNumber(103) var viewer_flags: Int? = null, @ProtoNumber(104) var history: List = emptyList(), @ProtoNumber(105) var updateStrategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE, + // syncyomi + @ProtoNumber(106) var lastModifiedAt: Long = 0, + @ProtoNumber(109) var version: Long = 0, // suwayomi @ProtoNumber(9000) var meta: Map = emptyMap(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt index 42f0a0862..1e7b34ca9 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt @@ -28,6 +28,7 @@ import kotlinx.coroutines.flow.update import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.sync.withPermit +import suwayomi.tachidesk.global.impl.sync.SyncManager import suwayomi.tachidesk.manga.impl.Category import suwayomi.tachidesk.manga.impl.CategoryManga import suwayomi.tachidesk.manga.impl.Chapter @@ -337,90 +338,98 @@ class Updater : IUpdater { clear: Boolean?, forceAll: Boolean, ) { - saveLastUpdateTimestamp() - - if (clear == true) { - reset() - } - - val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate } - val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.EXCLUDE].orEmpty() - val includedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.INCLUDE].orEmpty() - val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.UNSET].orEmpty() - val categoriesToUpdate = - if (forceAll) { - categories - } else { - includedCategories.ifEmpty { unsetCategories } - } - val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList() - val updateStatusCategories = - mapOf( - Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate), - Pair(CategoryUpdateStatus.SKIPPED, skippedCategories), - ) - - logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" } - - val categoriesToUpdateMangas = - categoriesToUpdate - .flatMap { CategoryManga.getCategoryMangaList(it.id) } - .distinctBy { it.id } - val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id }) - val mangasToUpdate = - categoriesToUpdateMangas - .asSequence() - .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE } - .filter { - if (serverConfig.excludeUnreadChapters.value) { - (it.unreadCount ?: 0L) == 0L - } else { - true - } - }.filter { - if (it.initialized && serverConfig.excludeNotStarted.value) { - it.lastReadAt != null - } else { - true - } - }.filter { - if (serverConfig.excludeCompleted.value) { - it.status != MangaStatus.COMPLETED.name - } else { - true - } - }.filter { forceAll || !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } } - .toList() - val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList() - - this.updateStatusCategories = updateStatusCategories - this.updateStatusSkippedMangas = skippedMangas - - if (mangasToUpdate.isEmpty()) { - // In case no manga gets updated and no update job was running before, the client would never receive an info - // about its update request - scope.launch { - updateStatus(immediate = true) - } - return - } - scope.launch { - updateStatus( - categoryUpdates = - updateStatusCategories[CategoryUpdateStatus.UPDATING] - ?.map { - CategoryUpdateJob(it, CategoryUpdateStatus.UPDATING) - }.orEmpty(), - mangaUpdates = mangasToUpdate.map { UpdateJob(it) }, - isRunning = true, + SyncManager.ensureSync() + + saveLastUpdateTimestamp() + + if (clear == true) { + reset() + } + + val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate } + val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.EXCLUDE].orEmpty() + val includedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.INCLUDE].orEmpty() + val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.UNSET].orEmpty() + val categoriesToUpdate = + if (forceAll) { + categories + } else { + includedCategories.ifEmpty { unsetCategories } + } + val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList() + val updateStatusCategories = + mapOf( + Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate), + Pair(CategoryUpdateStatus.SKIPPED, skippedCategories), + ) + + logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" } + + val categoriesToUpdateMangas = + categoriesToUpdate + .flatMap { CategoryManga.getCategoryMangaList(it.id) } + .distinctBy { it.id } + val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id }) + val mangasToUpdate = + categoriesToUpdateMangas + .asSequence() + .filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE } + .filter { + if (serverConfig.excludeUnreadChapters.value) { + (it.unreadCount ?: 0L) == 0L + } else { + true + } + }.filter { + if (it.initialized && serverConfig.excludeNotStarted.value) { + it.lastReadAt != null + } else { + true + } + }.filter { + if (serverConfig.excludeCompleted.value) { + it.status != MangaStatus.COMPLETED.name + } else { + true + } + }.filter { + forceAll || + !excludedCategories.any { category -> + mangasToCategoriesMap[it.id]?.contains(category) == true + } + }.toList() + val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList() + + this@Updater.updateStatusCategories = updateStatusCategories + this@Updater.updateStatusSkippedMangas = skippedMangas + + if (mangasToUpdate.isEmpty()) { + // In case no manga gets updated and no update job was running before, the client would never receive an info + // about its update request + scope.launch { + updateStatus(immediate = true) + } + return@launch + } + + scope.launch { + updateStatus( + categoryUpdates = + updateStatusCategories[CategoryUpdateStatus.UPDATING] + ?.map { + CategoryUpdateJob(it, CategoryUpdateStatus.UPDATING) + }.orEmpty(), + mangaUpdates = mangasToUpdate.map { UpdateJob(it) }, + isRunning = true, + ) + } + + addMangasToQueue( + mangasToUpdate + .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)), ) } - - addMangasToQueue( - mangasToUpdate - .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)), - ) } override fun addMangasToQueue(mangas: List) { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/CategoryDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/CategoryDataClass.kt index 0d6d131c2..353d87c79 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/CategoryDataClass.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/CategoryDataClass.kt @@ -30,5 +30,8 @@ data class CategoryDataClass( val size: Int, val includeInUpdate: IncludeOrExclude, val includeInDownload: IncludeOrExclude, + val version: Long, + val uid: Long, + val lastModifiedAt: Long, val meta: Map = emptyMap(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt index 85dc8e728..00bfeb590 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt @@ -38,6 +38,8 @@ data class ChapterDataClass( val pageCount: Int = -1, /** total chapter count, used to calculate if there's a next and prev chapter */ val chapterCount: Int? = null, + val lastModifiedAt: Long = 0, + val version: Long = 0, /** used to store client specific values */ val meta: Map = emptyMap(), ) { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt index 9af2eeba9..ad53be523 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt @@ -43,6 +43,8 @@ data class MangaDataClass( val age: Long? = if (lastFetchedAt == null) 0 else Instant.now().epochSecond.minus(lastFetchedAt), val chaptersAge: Long? = if (chaptersLastFetchedAt == null) null else Instant.now().epochSecond.minus(chaptersLastFetchedAt), val trackers: List? = null, + val lastModifiedAt: Long = 0, + val version: Long = 0, ) { override fun toString(): String = "\"$title\" (id= $id) (sourceId= $sourceId)" } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/CategoryTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/CategoryTable.kt index e234467b3..2f5459a44 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/CategoryTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/CategoryTable.kt @@ -19,6 +19,11 @@ object CategoryTable : IntIdTable() { val isDefault = bool("is_default").default(false) val includeInUpdate = integer("include_in_update").default(IncludeOrExclude.UNSET.value) val includeInDownload = integer("include_in_download").default(IncludeOrExclude.UNSET.value) + + val version = long("version").default(0) + val uid = long("uid").default(0) + val lastModifiedAt = long("last_modified_at").default(0) + val isSyncing = bool("is_syncing").default(false) } fun CategoryTable.toDataClass(categoryEntry: ResultRow) = @@ -30,5 +35,8 @@ fun CategoryTable.toDataClass(categoryEntry: ResultRow) = Category.getCategorySize(categoryEntry[id].value), IncludeOrExclude.fromValue(categoryEntry[includeInUpdate]), IncludeOrExclude.fromValue(categoryEntry[includeInDownload]), + categoryEntry[version], + categoryEntry[uid], + categoryEntry[lastModifiedAt], Category.getCategoryMetaMap(categoryEntry[id].value), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt index 735174297..e2f667723 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt @@ -42,6 +42,10 @@ object ChapterTable : IntIdTable() { val manga = reference("manga", MangaTable, ReferenceOption.CASCADE) val koreaderHash = varchar("koreader_hash", 32).nullable() + + val lastModifiedAt = long("last_modified_at").default(0) + val version = long("version").default(0) + val isSyncing = bool("is_syncing").default(false) } fun ChapterTable.toDataClass( @@ -83,4 +87,6 @@ fun ChapterTable.toDataClass( } else { emptyMap() }, + lastModifiedAt = chapterEntry[lastModifiedAt], + version = chapterEntry[version], ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt index 06c13931e..24412a8ac 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt @@ -46,6 +46,10 @@ object MangaTable : IntIdTable() { val chaptersLastFetchedAt = long("chapters_last_fetched_at").default(0) val updateStrategy = varchar("update_strategy", 256).default(UpdateStrategy.ALWAYS_UPDATE.name) + + val lastModifiedAt = long("last_modified_at").default(0) + val version = long("version").default(0) + val isSyncing = bool("is_syncing").default(false) } fun MangaTable.toDataClass( @@ -76,6 +80,8 @@ fun MangaTable.toDataClass( lastFetchedAt = mangaEntry[lastFetchedAt], chaptersLastFetchedAt = mangaEntry[chaptersLastFetchedAt], updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy]), + lastModifiedAt = mangaEntry[lastModifiedAt], + version = mangaEntry[version], ) enum class MangaStatus( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt index 04d446448..e637cc469 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/ServerSetup.kt @@ -36,6 +36,7 @@ import org.koin.core.context.startKoin import org.koin.core.module.Module import org.koin.dsl.module import suwayomi.tachidesk.global.impl.KcefWebView.Companion.toCefCookie +import suwayomi.tachidesk.global.impl.sync.SyncManager import suwayomi.tachidesk.graphql.types.DatabaseType import suwayomi.tachidesk.i18n.LocalizationHelper import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport @@ -517,6 +518,8 @@ fun applicationSetup() { // start DownloadManager and restore + resume downloads DownloadManager.restoreAndResumeDownloads() + SyncManager.scheduleSyncTask() + // asynchronously initialize CEF GlobalScope.launch { CEFManager.init() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0056_SyncYomi.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0056_SyncYomi.kt new file mode 100644 index 000000000..d6f668368 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0056_SyncYomi.kt @@ -0,0 +1,224 @@ +package suwayomi.tachidesk.server.database.migration + +import de.neonew.exposed.migrations.helpers.SQLMigration +import suwayomi.tachidesk.graphql.types.DatabaseType +import suwayomi.tachidesk.server.serverConfig + +@Suppress("ClassName", "unused") +class M0056_SyncYomi : SQLMigration() { + override val sql = + when (serverConfig.databaseType.value) { + DatabaseType.POSTGRESQL -> postgresQuery() + DatabaseType.H2 -> h2Query() + } + + // language=postgresql + fun postgresQuery(): String = + """ + ALTER TABLE manga ADD COLUMN version BIGINT NOT NULL DEFAULT 0; + ALTER TABLE manga ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE; + ALTER TABLE manga ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0; + + ALTER TABLE chapter ADD COLUMN version BIGINT NOT NULL DEFAULT 0; + ALTER TABLE chapter ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE; + ALTER TABLE chapter ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0; + + ALTER TABLE category ADD COLUMN version BIGINT NOT NULL DEFAULT 0; + ALTER TABLE category ADD COLUMN uid BIGINT NOT NULL DEFAULT 0; + ALTER TABLE category ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE; + ALTER TABLE category ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0; + + + CREATE OR REPLACE FUNCTION update_manga_version() + RETURNS trigger AS $$ + BEGIN + IF NOT NEW.is_syncing + AND ROW(NEW.url, NEW.description, NEW.in_library) + IS DISTINCT FROM + ROW(OLD.url, OLD.description, OLD.in_library) + THEN + NEW.version := OLD.version + 1; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER update_manga_version + AFTER UPDATE ON manga + FOR EACH ROW + EXECUTE FUNCTION update_manga_version(); + + + CREATE OR REPLACE FUNCTION update_chapter_and_manga_version() + RETURNS trigger AS $$ + BEGIN + IF NOT NEW.is_syncing + AND ROW(NEW.read, NEW.bookmark, NEW.last_page_read) + IS DISTINCT FROM + ROW(OLD.read, OLD.bookmark, OLD.last_page_read) + THEN + NEW.version := OLD.version + 1; + + UPDATE manga SET version = version + 1 WHERE id = NEW.manga AND is_syncing = FALSE; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER update_chapter_and_manga_version + AFTER UPDATE ON chapter + FOR EACH ROW + EXECUTE FUNCTION update_chapter_and_manga_version(); + + + CREATE OR REPLACE FUNCTION update_manga_last_modified_at() + RETURNS trigger AS $$ + BEGIN + NEW.last_modified_at := EXTRACT(EPOCH FROM NOW()); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER update_manga_last_modified_at + BEFORE UPDATE OR INSERT ON manga + FOR EACH ROW + EXECUTE FUNCTION update_manga_last_modified_at(); + + + CREATE OR REPLACE FUNCTION update_chapter_last_modified_at() + RETURNS trigger AS $$ + BEGIN + NEW.last_modified_at := EXTRACT(EPOCH FROM NOW()); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER update_chapter_last_modified_at + BEFORE UPDATE OR INSERT ON chapter + FOR EACH ROW + EXECUTE FUNCTION update_chapter_last_modified_at(); + + + CREATE OR REPLACE FUNCTION insert_manga_category_update_version() + RETURNS trigger AS $$ + BEGIN + UPDATE manga SET version = version + 1 WHERE id = NEW.manga AND is_syncing = FALSE; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER insert_manga_category_update_version + AFTER INSERT ON categorymanga + FOR EACH ROW + EXECUTE FUNCTION insert_manga_category_update_version(); + + + CREATE OR REPLACE FUNCTION insert_category_uid() + RETURNS trigger AS $$ + BEGIN + IF NEW.uid = 0 THEN + NEW.uid := RANDOM(1, 9223372036854775807); + END IF; + + IF NEW.last_modified_at = 0 THEN + NEW.last_modified_at := EXTRACT(EPOCH FROM NOW()); + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER insert_category_uid + BEFORE INSERT ON category + FOR EACH ROW + EXECUTE FUNCTION insert_category_uid(); + + + CREATE OR REPLACE FUNCTION update_category_version() + RETURNS trigger AS $$ + BEGIN + IF NOT NEW.is_syncing + AND ROW(NEW.name, NEW.sort_order) + IS DISTINCT FROM + ROW(OLD.name, OLD.sort_order) + THEN + NEW.version := NEW.version + 1; + NEW.last_modified_at := EXTRACT(EPOCH FROM NOW()); + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER update_category_version + BEFORE UPDATE ON category + FOR EACH ROW + EXECUTE FUNCTION update_category_version(); + """.trimIndent() + + // language=h2 + fun h2Query() = + """ + ALTER TABLE manga ADD COLUMN version BIGINT NOT NULL DEFAULT 0; + ALTER TABLE manga ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE; + ALTER TABLE manga ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0; + + ALTER TABLE chapter ADD COLUMN version BIGINT NOT NULL DEFAULT 0; + ALTER TABLE chapter ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE; + ALTER TABLE chapter ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0; + + ALTER TABLE category ADD COLUMN version BIGINT NOT NULL DEFAULT 0; + ALTER TABLE category ADD COLUMN uid BIGINT NOT NULL DEFAULT 0; + ALTER TABLE category ADD COLUMN is_syncing BOOLEAN NOT NULL DEFAULT FALSE; + ALTER TABLE category ADD COLUMN last_modified_at BIGINT NOT NULL DEFAULT 0; + + + CREATE TRIGGER update_manga_version + BEFORE UPDATE ON manga + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateMangaVersionTrigger"; + + CREATE TRIGGER update_chapter_and_manga_version + BEFORE UPDATE ON chapter + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateChapterAndMangaVersionTrigger"; + + CREATE TRIGGER update_manga_last_modified_at + BEFORE UPDATE ON manga + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateMangaLastModifiedAtTrigger"; + + CREATE TRIGGER insert_manga_last_modified_at + BEFORE INSERT ON manga + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateMangaLastModifiedAtTrigger"; + + CREATE TRIGGER update_chapter_last_modified_at + BEFORE UPDATE ON chapter + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateChapterLastModifiedAtTrigger"; + + CREATE TRIGGER insert_chapter_last_modified_at + BEFORE INSERT ON chapter + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateChapterLastModifiedAtTrigger"; + + CREATE TRIGGER insert_manga_category_update_version + AFTER INSERT ON categorymanga + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.InsertMangaCategoryUpdateVersionTrigger"; + + CREATE TRIGGER insert_category_uid + BEFORE INSERT ON category + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.InsertCategoryUidTrigger"; + + CREATE TRIGGER update_category_version + BEFORE UPDATE ON category + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateCategoryVersionTrigger"; + """.trimIndent() +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt new file mode 100644 index 000000000..8f483ad9d --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt @@ -0,0 +1,141 @@ +package suwayomi.tachidesk.server.database.trigger + +import org.h2.tools.TriggerAdapter +import java.sql.Connection +import java.sql.ResultSet +import kotlin.random.Random +import kotlin.time.Clock + +@Suppress("unused") +class UpdateMangaVersionTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet, + newRow: ResultSet, + ) { + val isSyncing = newRow.getBoolean("is_syncing") + val hasChanged = + oldRow.getString("url") != newRow.getString("url") || + oldRow.getString("description") != newRow.getString("description") || + oldRow.getBoolean("in_library") != newRow.getBoolean("in_library") + + if (!isSyncing && hasChanged) { + val currentVersion = newRow.getLong("version") + newRow.updateLong("version", currentVersion + 1) + } + } +} + +@Suppress("unused") +class UpdateChapterAndMangaVersionTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet, + newRow: ResultSet, + ) { + val isSyncing = newRow.getBoolean("is_syncing") + val hasChanged = + oldRow.getBoolean("read") != newRow.getBoolean("read") || + oldRow.getBoolean("bookmark") != newRow.getBoolean("bookmark") || + oldRow.getInt("last_page_read") != newRow.getInt("last_page_read") + + if (!isSyncing && hasChanged) { + val currentVersion = newRow.getLong("version") + newRow.updateLong("version", currentVersion + 1) + + val mangaId = newRow.getInt("manga") + conn + .prepareStatement( + "UPDATE MANGA SET version = version + 1 WHERE id = ? AND NOT is_syncing", + ).use { + it.setInt(1, mangaId) + it.executeUpdate() + } + } + } +} + +@Suppress("unused") +class UpdateMangaLastModifiedAtTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet?, + newRow: ResultSet, + ) { + newRow.updateLong("last_modified_at", Clock.System.now().epochSeconds) + } +} + +@Suppress("unused") +class UpdateChapterLastModifiedAtTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet?, + newRow: ResultSet, + ) { + newRow.updateLong("last_modified_at", Clock.System.now().epochSeconds) + } +} + +@Suppress("unused") +class InsertMangaCategoryUpdateVersionTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet?, + newRow: ResultSet, + ) { + val mangaId = newRow.getInt("manga") + + conn + .prepareStatement( + "UPDATE MANGA SET version = version + 1 WHERE id = ? AND NOT is_syncing", + ).use { + it.setInt(1, mangaId) + it.executeUpdate() + } + } +} + +@Suppress("unused") +class InsertCategoryUidTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet?, + newRow: ResultSet, + ) { + if (newRow.getLong("uid") == 0L) { + newRow.updateLong("uid", Random.nextLong(1, Long.MAX_VALUE)) + } + + if (newRow.getLong("last_modified_at") == 0L) { + newRow.updateLong( + "last_modified_at", + Clock.System.now().epochSeconds, + ) + } + } +} + +@Suppress("unused") +class UpdateCategoryVersionTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet, + newRow: ResultSet, + ) { + val isSyncing = newRow.getBoolean("is_syncing") + val hasChanged = + oldRow.getString("name") != newRow.getString("name") || + oldRow.getInt("sort_order") != newRow.getInt("sort_order") + + if (!isSyncing && hasChanged) { + val currentVersion = newRow.getLong("version") + newRow.updateLong("version", currentVersion + 1) + + newRow.updateLong( + "last_modified_at", + Clock.System.now().epochSeconds, + ) + } + } +}