diff --git a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt index 6f94b4c5d..4adf48213 100644 --- a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt +++ b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/ServerConfig.kt @@ -1016,6 +1016,26 @@ class ServerConfig( description = "Use Hikari Connection Pool to connect to the database.", ) + val syncYomiEnabled: MutableStateFlow by BooleanSetting( + protoNumber = 86, + defaultValue = false, + group = SettingGroup.SYNCYOMI, + privacySafe = true + ) + + val syncYomiHost: MutableStateFlow by StringSetting( + protoNumber = 87, + defaultValue = "", + group = SettingGroup.SYNCYOMI, + privacySafe = true, + ) + + val syncYomiApiKey: MutableStateFlow by StringSetting( + protoNumber = 88, + defaultValue = "", + group = SettingGroup.SYNCYOMI, + privacySafe = false, + ) /** ****************************************************************** **/ diff --git a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt index 5a329e3db..c9feff127 100644 --- a/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt +++ b/server/server-config/src/main/kotlin/suwayomi/tachidesk/server/settings/SettingGroup.kt @@ -17,6 +17,7 @@ enum class SettingGroup( CLOUDFLARE("Cloudflare"), OPDS("OPDS"), KOREADER_SYNC("KOReader sync"), + SYNCYOMI("SyncYomi") ; override fun toString(): String = value diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt new file mode 100644 index 000000000..cdba90822 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt @@ -0,0 +1,298 @@ +package suwayomi.tachidesk.global.impl.sync + +import android.app.Application +import android.content.Context +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.serialization.Serializable +import kotlinx.serialization.protobuf.ProtoBuf +import org.jetbrains.exposed.sql.selectAll +import org.jetbrains.exposed.sql.transactions.transaction +import org.jetbrains.exposed.sql.update +import suwayomi.tachidesk.manga.impl.backup.BackupFlags +import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupCategoryHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler +import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler +import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga +import suwayomi.tachidesk.manga.model.dataclass.ChapterDataClass +import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass +import suwayomi.tachidesk.manga.model.table.CategoryMangaTable +import suwayomi.tachidesk.manga.model.table.CategoryTable +import suwayomi.tachidesk.manga.model.table.ChapterTable +import suwayomi.tachidesk.manga.model.table.MangaStatus +import suwayomi.tachidesk.manga.model.table.MangaTable +import suwayomi.tachidesk.manga.model.table.toDataClass +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import java.util.Date +import kotlin.system.measureTimeMillis + +@Serializable +data class SyncData( + val backup: Backup? = null, +) + +object SyncManager { + private val syncPreferences = Injekt.get().getSharedPreferences("sync", Context.MODE_PRIVATE) + private val logger = KotlinLogging.logger {} + + suspend fun syncData() { + transaction { + MangaTable.update({ MangaTable.isSyncing eq true }) { + it[isSyncing] = false + } + ChapterTable.update({ ChapterTable.isSyncing eq true }) { + it[isSyncing] = false + } + } + + val databaseManga = getAllMangaThatNeedsSync() + + val backupFlags = BackupFlags( + includeManga = true, + includeCategories = true, + includeChapters = true, + includeTracking = true, + includeHistory = true, + includeClientData = false, + includeServerSettings = false, + ) + + val backupMangas = BackupMangaHandler.backup(backupFlags) + + val backup = Backup( + BackupMangaHandler.backup(backupFlags), + BackupCategoryHandler.backup(backupFlags), + BackupSourceHandler.backup(backupMangas, backupFlags), + emptyMap(), + null, + ) + + val syncData = SyncData( + backup = backup, + ) + + val remoteBackup = SyncYomiSyncService().doSync(syncData) + + if (remoteBackup == null) { + logger.debug { "Skip restore due to network issues" } + // should we call showSyncError? + return + } + + if (remoteBackup === syncData.backup) { + // nothing changed + logger.debug { "Skip restore due to remote was overwrite from local" } + syncPreferences.edit() + .putLong("last_sync_timestamp", Date().time) + .apply() + return + } + + // Stop the sync early if the remote backup is null or empty + if (remoteBackup.backupManga.isEmpty()) { + return + } + + // Check if it's first sync based on lastSyncTimestamp + if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && databaseManga.isNotEmpty()) { + // It's first sync no need to restore data. (just update remote data) + syncPreferences.edit() + .putLong("last_sync_timestamp", Date().time) + .apply() + return + } + + val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup) + updateNonFavorites(nonFavorites) + + val newSyncData = backup.copy( + backupManga = filteredFavorites, + backupCategories = remoteBackup.backupCategories, + backupSources = remoteBackup.backupSources, + ) + + // It's local sync no need to restore data. (just update remote data) + if (filteredFavorites.isEmpty()) { + // update the sync timestamp + syncPreferences.edit() + .putLong("last_sync_timestamp", Date().time) + .apply() + return + } + + val backupStream = ProtoBuf.encodeToByteArray(Backup.serializer(), newSyncData).inputStream() + ProtoBackupImport.restore( + sourceStream = backupStream, + flags = BackupFlags( + includeManga = true, + includeCategories = true, + includeChapters = true, + includeTracking = true, + includeHistory = true, + includeClientData = false, + includeServerSettings = false, + ), + isSync = true, + ) + + // update the sync timestamp + syncPreferences.edit() + .putLong("last_sync_timestamp", Date().time) + .apply() + } + + private fun getAllMangaFromDB(): List { + return transaction { MangaTable.selectAll().map { MangaTable.toDataClass(it) } } + } + + private fun getAllMangaThatNeedsSync(): List { + return transaction { + MangaTable.selectAll().where { MangaTable.inLibrary eq true }.map { MangaTable.toDataClass(it) } + } + } + + private fun isMangaDifferent(localManga: MangaDataClass, remoteManga: BackupManga): Boolean { + val localChapters = transaction { + ChapterTable + .selectAll() + .where { ChapterTable.manga eq localManga.id } + .map { ChapterTable.toDataClass(it) } + } + val localCategories = transaction { + CategoryMangaTable + .innerJoin(CategoryTable) + .selectAll() + .where { CategoryMangaTable.manga eq localManga.id } + .map { it[CategoryTable.order] } + } + + if (areChaptersDifferent(localChapters, remoteManga.chapters)) { + return true + } + + if (localManga.version != remoteManga.version) { + return true + } + + if (localCategories.toSet() != remoteManga.categories.toSet()) { + return true + } + + return false + } + + private fun areChaptersDifferent( + localChapters: List, + remoteChapters: List, + ): Boolean { + val localChapterMap = localChapters.associateBy { it.url } + val remoteChapterMap = remoteChapters.associateBy { it.url } + + if (localChapterMap.size != remoteChapterMap.size) { + return true + } + + for ((url, localChapter) in localChapterMap) { + val remoteChapter = remoteChapterMap[url] + + // If a matching remote chapter doesn't exist, or the version numbers are different, consider them different + if (remoteChapter == null || localChapter.version != remoteChapter.version) { + return true + } + } + + return false + } + + private fun filterFavoritesAndNonFavorites(backup: Backup): Pair, List> { + val favorites = mutableListOf() + val nonFavorites = mutableListOf() + + val elapsedTimeMillis = measureTimeMillis { + val databaseManga = getAllMangaFromDB() + val localMangaMap = databaseManga.associateBy { + Triple(it.sourceId.toLong(), it.url, it.title) + } + + logger.debug { "Starting to filter favorites and non-favorites from backup data." } + + backup.backupManga.forEach { remoteManga -> + val compositeKey = Triple(remoteManga.source, remoteManga.url, remoteManga.title) + val localManga = localMangaMap[compositeKey] + when { + // Checks if the manga is in favorites and needs updating or adding + remoteManga.favorite -> { + if (localManga == null || isMangaDifferent(localManga, remoteManga)) { + logger.debug { "Adding to favorites: ${remoteManga.title}" } + favorites.add(remoteManga) + } else { + logger.debug { "Already up-to-date favorite: ${remoteManga.title}" } + } + } + // Handle non-favorites + !remoteManga.favorite -> { + logger.debug { "Adding to non-favorites: ${remoteManga.title}" } + nonFavorites.add(remoteManga) + } + } + } + } + + val minutes = elapsedTimeMillis / 60000 + val seconds = (elapsedTimeMillis % 60000) / 1000 + logger.debug { "Filtering completed in ${minutes}m ${seconds}s. Favorites found: ${favorites.size}, Non-favorites found: ${nonFavorites.size}" } + + return Pair(favorites, nonFavorites) + } + + private fun updateNonFavorites(nonFavorites: List) { + val localMangaList = getAllMangaFromDB() + + val localMangaMap = localMangaList.associateBy { Triple(it.sourceId.toLong(), it.url, it.title) } + + nonFavorites.forEach { nonFavorite -> + val key = Triple(nonFavorite.source, nonFavorite.url, nonFavorite.title) + localMangaMap[key]?.let { localManga -> + if (localManga.inLibrary != nonFavorite.favorite) { + val updatedManga = localManga.copy(inLibrary = nonFavorite.favorite) + updateManga(updatedManga) + } + } + } + } + + private fun updateManga(manga: MangaDataClass) { + transaction { + MangaTable.update({ MangaTable.id eq manga.id }) { + it[MangaTable.url] = manga.url + it[MangaTable.title] = manga.title + it[MangaTable.initialized] = manga.initialized + + it[MangaTable.artist] = manga.artist + it[MangaTable.author] = manga.author + it[MangaTable.description] = manga.description + it[MangaTable.genre] = manga.genre.joinToString(separator = ", ") + + it[MangaTable.status] = MangaStatus.valueOf(manga.status).value + it[MangaTable.thumbnail_url] = manga.thumbnailUrl + it[MangaTable.thumbnailUrlLastFetched] = manga.thumbnailUrlLastFetched + + it[MangaTable.inLibrary] = manga.inLibrary + it[MangaTable.inLibraryAt] = manga.inLibraryAt + + it[MangaTable.sourceReference] = manga.sourceId.toLongOrNull() ?: 0L + + it[MangaTable.realUrl] = manga.realUrl + it[MangaTable.lastFetchedAt] = manga.lastFetchedAt ?: 0L + it[MangaTable.chaptersLastFetchedAt] = manga.chaptersLastFetchedAt ?: 0L + + it[MangaTable.updateStrategy] = manga.updateStrategy.name + + it[MangaTable.version] = manga.version + } + } + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt new file mode 100644 index 000000000..150240700 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt @@ -0,0 +1,401 @@ +package suwayomi.tachidesk.global.impl.sync + +import android.app.Application +import android.content.Context +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.PUT +import eu.kanade.tachiyomi.network.await +import io.github.oshai.kotlinlogging.KotlinLogging +import io.javalin.http.HttpStatus +import kotlinx.serialization.SerializationException +import kotlinx.serialization.protobuf.ProtoBuf +import okhttp3.Headers +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.RequestBody.Companion.toRequestBody +import suwayomi.tachidesk.manga.impl.Category +import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupCategory +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupChapter +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupManga +import suwayomi.tachidesk.manga.impl.backup.proto.models.BackupSource +import suwayomi.tachidesk.server.serverConfig +import uy.kohesive.injekt.Injekt +import uy.kohesive.injekt.api.get +import java.util.concurrent.TimeUnit + +class SyncYomiSyncService { + private val syncPreferences = Injekt.get().getSharedPreferences("sync", Context.MODE_PRIVATE) + + private val logger = KotlinLogging.logger {} + + private class SyncYomiException(message: String?) : Exception(message) + + suspend fun doSync(syncData: SyncData): Backup? { + try { + val (remoteData, etag) = pullSyncData() + + val finalSyncData = if (remoteData != null) { + assert(etag.isNotEmpty()) { "ETag should never be empty if remote data is not null" } + logger.debug { "Try update remote data with ETag($etag)" } + mergeSyncData(syncData, remoteData) + } else { + // init or overwrite remote data + logger.debug { "Try overwrite remote data with ETag($etag)" } + syncData + } + + pushSyncData(finalSyncData, etag) + return finalSyncData.backup + } catch (e: Exception) { + logger.error { "Error syncing: ${e.message}" } + return null + } + } + + private suspend fun pullSyncData(): Pair { + val host = serverConfig.syncYomiHost.value + val apiKey = serverConfig.syncYomiApiKey.value + val downloadUrl = "$host/api/sync/content" + + val headersBuilder = Headers.Builder().add("X-API-Token", apiKey) + val lastETag = syncPreferences.getString("last_sync_etag", "") ?: "" + if (lastETag != "") { + headersBuilder.add("If-None-Match", lastETag) + } + val headers = headersBuilder.build() + + val downloadRequest = GET( + url = downloadUrl, + headers = headers, + ) + + val client = OkHttpClient() + val response = client.newCall(downloadRequest).await() + + if (response.code == HttpStatus.NOT_MODIFIED.code) { + // not modified + assert(lastETag.isNotEmpty()) + logger.info { "Remote server not modified" } + return Pair(null, lastETag) + } else if (response.code == HttpStatus.NOT_FOUND.code) { + // maybe got deleted from remote + return Pair(null, "") + } + + if (response.isSuccessful) { + val newETag = response.headers["ETag"] + .takeIf { it?.isNotEmpty() == true } ?: throw SyncYomiException("Missing ETag") + + val byteArray = response.body.byteStream().use { + return@use it.readBytes() + } + + return try { + val backup = ProtoBuf.decodeFromByteArray(Backup.serializer(), byteArray) + return Pair(SyncData(backup = backup), newETag) + } catch (_: SerializationException) { + logger.info { "Bad content responsed from server" } + // the body is invalid + // return default value so we can overwrite it + Pair(null, "") + } + } else { + val responseBody = response.body.string() + logger.error { "SyncError: $responseBody" } + throw SyncYomiException("Failed to download sync data: $responseBody") + } + } + + private suspend fun pushSyncData(syncData: SyncData, eTag: String) { + val backup = syncData.backup ?: return + + val host = serverConfig.syncYomiHost.value + val apiKey = serverConfig.syncYomiApiKey.value + val uploadUrl = "$host/api/sync/content" + val timeout = 30L + + val headersBuilder = Headers.Builder().add("X-API-Token", apiKey) + if (eTag.isNotEmpty()) { + headersBuilder.add("If-Match", eTag) + } + val headers = headersBuilder.build() + + // Set timeout to 30 seconds + val client = OkHttpClient.Builder() + .connectTimeout(timeout, TimeUnit.SECONDS) + .readTimeout(timeout, TimeUnit.SECONDS) + .writeTimeout(timeout, TimeUnit.SECONDS) + .build() + + val byteArray = ProtoBuf.encodeToByteArray(Backup.serializer(), backup) + if (byteArray.isEmpty()) { + throw IllegalStateException("Empty backup error") + } + val body = byteArray.toRequestBody("application/octet-stream".toMediaType()) + + val uploadRequest = PUT( + url = uploadUrl, + headers = headers, + body = body, + ) + + val response = client.newCall(uploadRequest).await() + + if (response.isSuccessful) { + val newETag = response.headers["ETag"] + .takeIf { it?.isNotEmpty() == true } ?: throw SyncYomiException("Missing ETag") + syncPreferences.edit() + .putString("last_sync_etag", newETag) + .apply() + logger.debug { "SyncYomi sync completed" } + } else if (response.code == HttpStatus.PRECONDITION_FAILED.code) { + // other clients updated remote data, will try next time + logger.debug { "SyncYomi sync failed with 412" } + } else { + val responseBody = response.body.string() + logger.error { "SyncError: $responseBody" } + } + } + + fun mergeSyncData(localSyncData: SyncData, remoteSyncData: SyncData): SyncData { + val mergedCategoriesList = + mergeCategoriesLists(localSyncData.backup?.backupCategories, remoteSyncData.backup?.backupCategories) + + val mergedMangaList = mergeMangaLists( + localSyncData.backup?.backupManga, + remoteSyncData.backup?.backupManga, + localSyncData.backup?.backupCategories ?: emptyList(), + remoteSyncData.backup?.backupCategories ?: emptyList(), + mergedCategoriesList, + ) + + val mergedSourcesList = + mergeSourcesLists(localSyncData.backup?.backupSources, remoteSyncData.backup?.backupSources) + + // Create the merged Backup object + val mergedBackup = Backup( + backupManga = mergedMangaList, + backupCategories = mergedCategoriesList, + backupSources = mergedSourcesList, + meta = emptyMap(), + serverSettings = null, + ) + + // Create the merged SData object + return SyncData( + backup = mergedBackup, + ) + } + + private fun mergeMangaLists( + localMangaList: List?, + remoteMangaList: List?, + localCategories: List, + remoteCategories: List, + mergedCategories: List, + ): List { + val localMangaListSafe = localMangaList.orEmpty() + val remoteMangaListSafe = remoteMangaList.orEmpty() + + logger.debug { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" } + + fun mangaCompositeKey(manga: BackupManga): String { + return "${manga.source}|${manga.url}|${manga.title.lowercase().trim()}|${manga.author?.lowercase()?.trim()}" + } + + // Create maps using composite keys + val localMangaMap = localMangaListSafe.associateBy { mangaCompositeKey(it) } + val remoteMangaMap = remoteMangaListSafe.associateBy { mangaCompositeKey(it) } + + val localCategoriesMapByOrder = localCategories.associateBy { it.order } + val remoteCategoriesMapByOrder = remoteCategories.associateBy { it.order } + val mergedCategoriesMapByName = mergedCategories.associateBy { it.name } + + fun updateCategories(theManga: BackupManga, theMap: Map): BackupManga { + return theManga.copy( + categories = theManga.categories.mapNotNull { + theMap[it]?.let { category -> + mergedCategoriesMapByName[category.name]?.order + } + }, + ) + } + + logger.debug { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" } + + val mergedList = (localMangaMap.keys + remoteMangaMap.keys).distinct().mapNotNull { compositeKey -> + val local = localMangaMap[compositeKey] + val remote = remoteMangaMap[compositeKey] + + // New version comparison logic + when { + local != null && remote == null -> updateCategories(local, localCategoriesMapByOrder) + local == null && remote != null -> updateCategories(remote, remoteCategoriesMapByOrder) + local != null && remote != null -> { + // Compare versions to decide which manga to keep + if (local.version >= remote.version) { + logger.debug { "Keeping local version of ${local.title} with merged chapters." } + updateCategories( + local.copy(chapters = mergeChapters(local.chapters, remote.chapters)), + localCategoriesMapByOrder, + ) + } else { + logger.debug { "Keeping remote version of ${remote.title} with merged chapters." } + updateCategories( + remote.copy(chapters = mergeChapters(local.chapters, remote.chapters)), + remoteCategoriesMapByOrder, + ) + } + } + + else -> null // No manga found for key + } + } + + // Counting favorites and non-favorites + val (favorites, nonFavorites) = mergedList.partition { it.favorite } + + logger.debug { "Merge completed. Total merged manga: ${mergedList.size}, Favorites: ${favorites.size}, Non-Favorites: ${nonFavorites.size}" } + + return mergedList + } + + private fun mergeChapters( + localChapters: List, + remoteChapters: List, + ): List { + fun chapterCompositeKey(chapter: BackupChapter): String { + return "${chapter.url}|${chapter.name}|${chapter.chapterNumber}" + } + + val localChapterMap = localChapters.associateBy { chapterCompositeKey(it) } + val remoteChapterMap = remoteChapters.associateBy { chapterCompositeKey(it) } + + logger.debug { "Starting chapter merge. Local chapters: ${localChapters.size}, Remote chapters: ${remoteChapters.size}" } + + // Merge both chapter maps based on version numbers + val mergedChapters = (localChapterMap.keys + remoteChapterMap.keys).distinct().mapNotNull { compositeKey -> + val localChapter = localChapterMap[compositeKey] + val remoteChapter = remoteChapterMap[compositeKey] + + logger.debug { "Processing chapter key: $compositeKey. Local chapter: ${localChapter != null}, Remote chapter: ${remoteChapter != null}" } + + when { + localChapter != null && remoteChapter == null -> { + logger.debug { "Keeping local chapter: ${localChapter.name}." } + localChapter + } + + localChapter == null && remoteChapter != null -> { + logger.debug { "Taking remote chapter: ${remoteChapter.name}." } + remoteChapter + } + + localChapter != null && remoteChapter != null -> { + // Use version number to decide which chapter to keep + val chosenChapter = if (localChapter.version >= remoteChapter.version) { + // If there mare more chapter on remote, local sourceOrder will need to be updated to maintain correct source order. + if (localChapters.size < remoteChapters.size) { + localChapter.copy(sourceOrder = remoteChapter.sourceOrder) + } else { + localChapter + } + } else { + remoteChapter + } + logger.debug { "Merging chapter: ${chosenChapter.name}. Chosen version from: ${if (localChapter.version >= remoteChapter.version) "Local" else "Remote"}, Local version: ${localChapter.version}, Remote version: ${remoteChapter.version}." } + chosenChapter + } + + else -> { + logger.debug { "No chapter found for composite key: $compositeKey. Skipping." } + null + } + } + } + + logger.debug { "Chapter merge completed. Total merged chapters: ${mergedChapters.size}" } + + return mergedChapters + } + + private fun mergeCategoriesLists( + localCategoriesList: List?, + remoteCategoriesList: List?, + ): List { + if (localCategoriesList == null) return remoteCategoriesList ?: emptyList() + if (remoteCategoriesList == null) return localCategoriesList + val localCategoriesMap = localCategoriesList + .filter { it.name != Category.DEFAULT_CATEGORY_NAME } + .associateBy { it.name } + val remoteCategoriesMap = remoteCategoriesList.associateBy { it.name } + + val mergedCategoriesMap = mutableMapOf() + + localCategoriesMap.forEach { (name, localCategory) -> + val remoteCategory = remoteCategoriesMap[name] + if (remoteCategory != null) { + // Compare and merge local and remote categories + val mergedCategory = if (localCategory.order > remoteCategory.order) { + localCategory + } else { + remoteCategory + } + mergedCategoriesMap[name] = mergedCategory + } else { + // If the category is only in the local list, add it to the merged list + mergedCategoriesMap[name] = localCategory + } + } + + // Add any categories from the remote list that are not in the local list + remoteCategoriesMap.forEach { (name, remoteCategory) -> + if (!mergedCategoriesMap.containsKey(name)) { + mergedCategoriesMap[name] = remoteCategory + } + } + + return mergedCategoriesMap.values.toList() + } + + private fun mergeSourcesLists( + localSources: List?, + remoteSources: List?, + ): List { + // Create maps using sourceId as key + val localSourceMap = localSources?.associateBy { it.sourceId } ?: emptyMap() + val remoteSourceMap = remoteSources?.associateBy { it.sourceId } ?: emptyMap() + + logger.debug { "Starting source merge. Local sources: ${localSources?.size}, Remote sources: ${remoteSources?.size}" } + + // Merge both source maps + val mergedSources = (localSourceMap.keys + remoteSourceMap.keys).distinct().mapNotNull { sourceId -> + val localSource = localSourceMap[sourceId] + val remoteSource = remoteSourceMap[sourceId] + + logger.debug { "Processing source ID: $sourceId. Local source: ${localSource != null}, Remote source: ${remoteSource != null}" } + + when { + localSource != null && remoteSource == null -> { + logger.debug { "Using local source: ${localSource.name}." } + localSource + } + + remoteSource != null && localSource == null -> { + logger.debug { "Using remote source: ${remoteSource.name}." } + remoteSource + } + + else -> { + logger.debug { "Remote and local is not empty: $sourceId. Skipping." } + null + } + } + } + + logger.debug { "Source merge completed. Total merged sources: ${mergedSources.size}" } + + return mergedSources + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt new file mode 100644 index 000000000..32cbf5ad8 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/SyncMutation.kt @@ -0,0 +1,34 @@ +package suwayomi.tachidesk.graphql.mutations + +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch +import suwayomi.tachidesk.global.impl.sync.SyncManager +import suwayomi.tachidesk.graphql.directives.RequireAuth +import suwayomi.tachidesk.server.serverConfig + +class SyncMutation { + data class StartSyncInput( + val clientMutationId: String? = null, + ) + + data class StartSyncPayload( + val clientMutationId: String? = null, + ) + + @OptIn(DelicateCoroutinesApi::class) + @RequireAuth + fun startSync(input: StartSyncInput): StartSyncPayload { + val (clientMutationId) = input + + if (serverConfig.syncYomiEnabled.value) { + GlobalScope.launch { + SyncManager.syncData() + } + } + + return StartSyncPayload( + clientMutationId = clientMutationId, + ) + } +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt index dd51ee552..8a0b2e75a 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt @@ -27,6 +27,7 @@ import suwayomi.tachidesk.graphql.mutations.MangaMutation import suwayomi.tachidesk.graphql.mutations.MetaMutation import suwayomi.tachidesk.graphql.mutations.SettingsMutation import suwayomi.tachidesk.graphql.mutations.SourceMutation +import suwayomi.tachidesk.graphql.mutations.SyncMutation import suwayomi.tachidesk.graphql.mutations.TrackMutation import suwayomi.tachidesk.graphql.mutations.UpdateMutation import suwayomi.tachidesk.graphql.mutations.UserMutation @@ -114,6 +115,7 @@ val schema = TopLevelObject(MangaMutation()), TopLevelObject(MetaMutation()), TopLevelObject(SettingsMutation()), + TopLevelObject(SyncMutation()), TopLevelObject(SourceMutation()), TopLevelObject(TrackMutation()), TopLevelObject(UpdateMutation()), diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt index b5f7d4154..4e7d1f271 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Chapter.kt @@ -125,6 +125,7 @@ object Chapter { downloaded = dbChapter[ChapterTable.isDownloaded], pageCount = dbChapter[ChapterTable.pageCount], chapterCount = chapterList.size, + version = dbChapter[ChapterTable.version], meta = chapterMetas.getValue(dbChapter[ChapterTable.id].value), ) } @@ -283,6 +284,7 @@ object Chapter { this[ChapterTable.isRead] = false this[ChapterTable.isBookmarked] = false this[ChapterTable.isDownloaded] = false + this[ChapterTable.version] = chapter.version // is recognized chapter number if (chapter.chapterNumber >= 0f && chapter.chapterNumber in deletedChapterNumbers) { @@ -315,6 +317,7 @@ object Chapter { this[ChapterTable.scanlator] = it.scanlator this[ChapterTable.sourceOrder] = it.index this[ChapterTable.realUrl] = it.realUrl + this[ChapterTable.version] = it.version } execute(this@transaction) } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt index a52c29d94..bab95b21a 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Manga.kt @@ -96,6 +96,7 @@ object Manga { updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]), freshData = true, trackers = Track.getTrackRecordsByMangaId(mangaId), + version = mangaEntry[MangaTable.version], ) } } @@ -243,6 +244,7 @@ object Manga { updateStrategy = UpdateStrategy.valueOf(mangaEntry[MangaTable.updateStrategy]), freshData = false, trackers = Track.getTrackRecordsByMangaId(mangaId), + version = mangaEntry[MangaTable.version], ) fun getMangaMetaMap(mangaId: Int): Map = diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt index a4f31f742..93591bfde 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt @@ -21,6 +21,8 @@ import kotlinx.coroutines.sync.withLock import okio.buffer import okio.gzip import okio.source +import org.jetbrains.exposed.sql.transactions.transaction +import org.jetbrains.exposed.sql.update import suwayomi.tachidesk.graphql.types.toStatus import suwayomi.tachidesk.manga.impl.backup.BackupFlags import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator.ValidationResult @@ -31,6 +33,8 @@ import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupMangaHandler import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSettingsHandler import suwayomi.tachidesk.manga.impl.backup.proto.handlers.BackupSourceHandler import suwayomi.tachidesk.manga.impl.backup.proto.models.Backup +import suwayomi.tachidesk.manga.model.table.ChapterTable +import suwayomi.tachidesk.manga.model.table.MangaTable import java.io.InputStream import java.util.Date import java.util.Timer @@ -109,6 +113,7 @@ object ProtoBackupImport : ProtoBackupBase() { fun restore( sourceStream: InputStream, flags: BackupFlags, + isSync: Boolean = false, ): String { val restoreId = System.currentTimeMillis().toString() @@ -117,7 +122,7 @@ object ProtoBackupImport : ProtoBackupBase() { updateRestoreState(restoreId, BackupRestoreState.Idle) GlobalScope.launch { - restoreLegacy(sourceStream, restoreId, flags) + restoreLegacy(sourceStream, restoreId, flags, isSync) } return restoreId @@ -127,11 +132,12 @@ object ProtoBackupImport : ProtoBackupBase() { sourceStream: InputStream, restoreId: String = "legacy", flags: BackupFlags = BackupFlags.DEFAULT, + isSync: Boolean = false, ): ValidationResult = backupMutex.withLock { try { logger.info { "restore($restoreId): restoring..." } - performRestore(restoreId, sourceStream, flags) + performRestore(restoreId, sourceStream, flags, isSync) } catch (e: Exception) { logger.error(e) { "restore($restoreId): failed due to" } @@ -152,11 +158,14 @@ object ProtoBackupImport : ProtoBackupBase() { id: String, sourceStream: InputStream, flags: BackupFlags, + isSync: Boolean, ): ValidationResult { val backupString = sourceStream .source() - .gzip() + .run { + if (!isSync) gzip() else this + } .buffer() .use { it.readByteArray() } val backup = parser.decodeFromByteArray(Backup.serializer(), backupString) @@ -235,6 +244,17 @@ object ProtoBackupImport : ProtoBackupBase() { """.trimIndent() } + if (isSync) { + transaction { + MangaTable.update({ MangaTable.isSyncing eq true}) { + it[isSyncing] = false + } + ChapterTable.update({ ChapterTable.isSyncing eq true}) { + it[isSyncing] = false + } + } + } + updateRestoreState(id, BackupRestoreState.Success) return validationResult diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt index 4c0f2bbbf..c5477891c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt @@ -73,6 +73,7 @@ object BackupMangaHandler { dateAdded = mangaRow[MangaTable.inLibraryAt].seconds.inWholeMilliseconds, viewer = 0, // not supported in Tachidesk updateStrategy = UpdateStrategy.valueOf(mangaRow[MangaTable.updateStrategy]), + version = mangaRow[MangaTable.version], ) val mangaId = mangaRow[MangaTable.id].value @@ -108,6 +109,7 @@ object BackupMangaHandler { it.uploadDate, it.chapterNumber, chapters.size - it.index, + it.version ).apply { if (flags.includeClientData) { this.meta = chapterToMeta[it.id] ?: emptyMap() @@ -230,6 +232,8 @@ object BackupMangaHandler { it[inLibrary] = manga.favorite it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds + + it[version] = manga.version }.value } else { val dbMangaId = dbManga[MangaTable.id].value @@ -249,6 +253,8 @@ object BackupMangaHandler { it[inLibrary] = manga.favorite || dbManga[inLibrary] it[inLibraryAt] = manga.dateAdded.milliseconds.inWholeSeconds + + it[version] = manga.version } dbMangaId @@ -337,6 +343,8 @@ object BackupMangaHandler { this[ChapterTable.lastReadAt] = historyByChapter[chapter.url]?.maxOrNull()?.milliseconds?.inWholeSeconds ?: 0 } + + this[ChapterTable.version] = chapter.version }.map { it[ChapterTable.id].value } } else { emptyList() diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt index 145184226..56cefa5e0 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupChapter.kt @@ -19,6 +19,8 @@ data class BackupChapter( // chapterNumber is called number is 1.x @ProtoNumber(9) var chapterNumber: Float = 0F, @ProtoNumber(10) var sourceOrder: Int = 0, + // syncyomi + @ProtoNumber(12) var version: Long = 0, // suwayomi @ProtoNumber(9000) var meta: Map = emptyMap(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt index c295c70c7..48d4fa082 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/models/BackupManga.kt @@ -34,6 +34,8 @@ data class BackupManga( @ProtoNumber(103) var viewer_flags: Int? = null, @ProtoNumber(104) var history: List = emptyList(), @ProtoNumber(105) var updateStrategy: UpdateStrategy = UpdateStrategy.ALWAYS_UPDATE, + // syncyomi + @ProtoNumber(109) var version: Long = 0, // suwayomi @ProtoNumber(9000) var meta: Map = emptyMap(), ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt index 85dc8e728..034dcd40c 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/ChapterDataClass.kt @@ -38,6 +38,7 @@ data class ChapterDataClass( val pageCount: Int = -1, /** total chapter count, used to calculate if there's a next and prev chapter */ val chapterCount: Int? = null, + val version: Long = 0, /** used to store client specific values */ val meta: Map = emptyMap(), ) { diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt index 9af2eeba9..470167cab 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/dataclass/MangaDataClass.kt @@ -43,6 +43,7 @@ data class MangaDataClass( val age: Long? = if (lastFetchedAt == null) 0 else Instant.now().epochSecond.minus(lastFetchedAt), val chaptersAge: Long? = if (chaptersLastFetchedAt == null) null else Instant.now().epochSecond.minus(chaptersLastFetchedAt), val trackers: List? = null, + val version: Long = 0, ) { override fun toString(): String = "\"$title\" (id= $id) (sourceId= $sourceId)" } diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt index 6b27729ad..cadc88722 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/ChapterTable.kt @@ -41,6 +41,9 @@ object ChapterTable : IntIdTable() { val manga = reference("manga", MangaTable, ReferenceOption.CASCADE) val koreaderHash = varchar("koreader_hash", 32).nullable() + + val version = long("version").default(0) + val isSyncing = bool("is_syncing").default(false) } fun ChapterTable.toDataClass( @@ -82,4 +85,5 @@ fun ChapterTable.toDataClass( } else { emptyMap() }, + version = chapterEntry[version], ) diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt index e17b73b7d..3209a5386 100644 --- a/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt +++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/model/table/MangaTable.kt @@ -46,6 +46,9 @@ object MangaTable : IntIdTable() { val chaptersLastFetchedAt = long("chapters_last_fetched_at").default(0) val updateStrategy = varchar("update_strategy", 256).default(UpdateStrategy.ALWAYS_UPDATE.name) + + val version = long("version").default(0) + val isSyncing = bool("is_syncing").default(false) } fun MangaTable.toDataClass( @@ -76,6 +79,7 @@ fun MangaTable.toDataClass( lastFetchedAt = mangaEntry[lastFetchedAt], chaptersLastFetchedAt = mangaEntry[chaptersLastFetchedAt], updateStrategy = UpdateStrategy.valueOf(mangaEntry[updateStrategy]), + version = mangaEntry[version], ) enum class MangaStatus( diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0053_SyncYomi.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0053_SyncYomi.kt new file mode 100644 index 000000000..b6e69e46c --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0053_SyncYomi.kt @@ -0,0 +1,29 @@ +package suwayomi.tachidesk.server.database.migration + +import de.neonew.exposed.migrations.helpers.SQLMigration + +@Suppress("ClassName", "unused") +class M0053_SyncYomi : SQLMigration() { + override val sql = """ + ALTER TABLE MANGA ADD COLUMN VERSION BIGINT DEFAULT 0; + ALTER TABLE MANGA ADD COLUMN IS_SYNCING BOOLEAN DEFAULT 0; + + ALTER TABLE CHAPTER ADD COLUMN VERSION BIGINT DEFAULT 0; + ALTER TABLE CHAPTER ADD COLUMN IS_SYNCING BOOLEAN DEFAULT 0; + + CREATE TRIGGER update_manga_version + AFTER UPDATE ON MANGA + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateMangaVersionTrigger"; + + CREATE TRIGGER update_chapter_and_manga_version + AFTER UPDATE ON CHAPTER + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.UpdateChapterAndMangaVersionTrigger"; + + CREATE TRIGGER insert_manga_category_update_version + AFTER INSERT ON CATEGORYMANGA + FOR EACH ROW + CALL "suwayomi.tachidesk.server.database.trigger.InsertMangaCategoryUpdateVersionTrigger"; + """.trimIndent() +} diff --git a/server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt b/server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt new file mode 100644 index 000000000..58cfa50e0 --- /dev/null +++ b/server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt @@ -0,0 +1,84 @@ +package suwayomi.tachidesk.server.database.trigger + +import org.h2.tools.TriggerAdapter +import java.sql.Connection +import java.sql.ResultSet + +@Suppress("unused") +class UpdateMangaVersionTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet, + newRow: ResultSet, + ) { + val isSyncing = newRow.getBoolean("is_syncing") + val hasChanged = oldRow.getString("url") != newRow.getString("url") || + oldRow.getString("description") != newRow.getString("description") || + oldRow.getBoolean("in_library") != newRow.getBoolean("in_library") + + if (!isSyncing && hasChanged) { + val id = newRow.getInt("id") + + conn.prepareStatement( + "UPDATE MANGA SET version = version + 1 WHERE id = ?", + ).use { + it.setInt(1, id) + it.executeUpdate() + } + } + } +} + +@Suppress("unused") +class UpdateChapterAndMangaVersionTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet, + newRow: ResultSet, + ) { + val isSyncing = newRow.getBoolean("is_syncing") + val hasChanged = oldRow.getBoolean("read") != newRow.getBoolean("read") || + oldRow.getBoolean("bookmark") != newRow.getBoolean("bookmark") || + oldRow.getInt("last_page_read") != newRow.getInt("last_page_read") + + if (!isSyncing && hasChanged) { + val chapterId = newRow.getInt("id") + val mangaId = newRow.getInt("manga") + + conn.prepareStatement( + "UPDATE CHAPTER SET version = version + 1 WHERE id = ?", + ).use { + it.setInt(1, chapterId) + it.executeUpdate() + } + + conn.prepareStatement( + "UPDATE MANGA SET version = version + 1 WHERE id = ? AND (SELECT is_syncing FROM MANGA WHERE id = ?) = FALSE", + ).use { + it.setInt(1, mangaId) + it.setInt(2, mangaId) + it.executeUpdate() + } + } + } +} + +@Suppress("unused") +class InsertMangaCategoryUpdateVersionTrigger : TriggerAdapter() { + override fun fire( + conn: Connection, + oldRow: ResultSet?, + newRow: ResultSet, + ) { + val mangaId = newRow.getInt("manga") + + conn.prepareStatement( + "UPDATE MANGA SET version = version + 1 WHERE id = ? AND (SELECT is_syncing FROM MANGA WHERE id = ?) = FALSE", + ).use { + it.setInt(1, mangaId) + it.setInt(2, mangaId) + it.executeUpdate() + } + } +} +