temple/Suwayomi-Server
1
0
Fork 0
mirror of https://github.com/Suwayomi/Suwayomi-Server.git synced 2026-07-16 01:04:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

Lint

Browse Source
This commit is contained in:
Bartu Özen
2025-12-20 20:09:48 +03:00
parent 17af9b6419
commit a4bfecc09c
7 changed files with 348 additions and 283 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt
View File

@@ -60,7 +60,8 @@ object SyncManager {
serverConfig.syncInterval,
) { enabled, interval -> Pair(enabled, interval) },
{ (enabled, interval) ->
currentTaskId = if (enabled && interval > 0) {
currentTaskId =
if (enabled && interval > 0) {
val intervalMs = interval.minutes.inWholeMilliseconds
currentTaskId?.let { HAScheduler.deschedule(it) }
@@ -132,7 +133,8 @@ object SyncManager {
val databaseManga = getAllMangaThatNeedsSync()
val backupFlags = BackupFlags(
val backupFlags =
BackupFlags(
includeManga = serverConfig.syncDataManga.value,
includeCategories = serverConfig.syncDataCategories.value,
includeChapters = serverConfig.syncDataChapters.value,
@@ -144,7 +146,8 @@ object SyncManager {
val backupMangas = BackupMangaHandler.backup(backupFlags)
val backup = Backup(
val backup =
Backup(
BackupMangaHandler.backup(backupFlags),
BackupCategoryHandler.backup(backupFlags).filter { it.name != Category.DEFAULT_CATEGORY_NAME },
BackupSourceHandler.backup(backupMangas, backupFlags),
@@ -152,7 +155,8 @@ object SyncManager {
null,
)
val syncData = SyncData(
val syncData =
SyncData(
backup = backup,
)
@@ -167,7 +171,8 @@ object SyncManager {
if (remoteBackup === syncData.backup) {
// nothing changed
logger.debug { "Skip restore due to remote was overwrite from local" }
syncPreferences.edit()
syncPreferences
.edit()
.putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
.apply()
return
@@ -181,7 +186,8 @@ object SyncManager {
// Check if it's first sync based on lastSyncTimestamp
if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && databaseManga.isNotEmpty()) {
// It's first sync no need to restore data. (just update remote data)
syncPreferences.edit()
syncPreferences
.edit()
.putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
.apply()
return
@@ -190,7 +196,8 @@ object SyncManager {
val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup)
updateNonFavorites(nonFavorites)
val newSyncData = backup.copy(
val newSyncData =
backup.copy(
backupManga = filteredFavorites,
backupCategories = remoteBackup.backupCategories,
backupSources = remoteBackup.backupSources,
@@ -199,7 +206,8 @@ object SyncManager {
// It's local sync no need to restore data. (just update remote data)
if (filteredFavorites.isEmpty()) {
// update the sync timestamp
syncPreferences.edit()
syncPreferences
.edit()
.putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
.apply()
return
@@ -208,7 +216,8 @@ object SyncManager {
val backupStream = ProtoBuf.encodeToByteArray(Backup.serializer(), newSyncData).inputStream()
ProtoBackupImport.restore(
sourceStream = backupStream,
flags = BackupFlags(
flags =
BackupFlags(
includeManga = true,
includeCategories = true,
includeChapters = true,
@@ -221,29 +230,32 @@ object SyncManager {
)
// update the sync timestamp
syncPreferences.edit()
syncPreferences
.edit()
.putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
.apply()
}
private fun getAllMangaFromDB(): List<MangaDataClass> {
return transaction { MangaTable.selectAll().map { MangaTable.toDataClass(it) } }
}
private fun getAllMangaFromDB(): List<MangaDataClass> = transaction { MangaTable.selectAll().map { MangaTable.toDataClass(it) } }
private fun getAllMangaThatNeedsSync(): List<MangaDataClass> {
return transaction {
private fun getAllMangaThatNeedsSync(): List<MangaDataClass> =
transaction {
MangaTable.selectAll().where { MangaTable.inLibrary eq true }.map { MangaTable.toDataClass(it) }
}
}
private fun isMangaDifferent(localManga: MangaDataClass, remoteManga: BackupManga): Boolean {
val localChapters = transaction {
private fun isMangaDifferent(
localManga: MangaDataClass,
remoteManga: BackupManga,
): Boolean {
val localChapters =
transaction {
ChapterTable
.selectAll()
.where { ChapterTable.manga eq localManga.id }
.map { ChapterTable.toDataClass(it) }
}
val localCategories = transaction {
val localCategories =
transaction {
CategoryMangaTable
.innerJoin(CategoryTable)
.selectAll()
@@ -293,9 +305,11 @@ object SyncManager {
val favorites = mutableListOf<BackupManga>()
val nonFavorites = mutableListOf<BackupManga>()
val elapsedTimeMillis = measureTimeMillis {
val elapsedTimeMillis =
measureTimeMillis {
val databaseManga = getAllMangaFromDB()
val localMangaMap = databaseManga.associateBy {
val localMangaMap =
databaseManga.associateBy {
Triple(it.sourceId.toLong(), it.url, it.title)
}
@@ -314,6 +328,7 @@ object SyncManager {
logger.debug { "Already up-to-date favorite: ${remoteManga.title}" }
}
}
// Handle non-favorites
!remoteManga.favorite -> {
logger.debug { "Adding to non-favorites: ${remoteManga.title}" }
@@ -325,7 +340,9 @@ object SyncManager {
val minutes = elapsedTimeMillis / 60000
val seconds = (elapsedTimeMillis % 60000) / 1000
logger.debug { "Filtering completed in ${minutes}m ${seconds}s. Favorites found: ${favorites.size}, Non-favorites found: ${nonFavorites.size}" }
logger.debug {
"Filtering completed in ${minutes}m ${seconds}s. Favorites found: ${favorites.size}, Non-favorites found: ${nonFavorites.size}"
}
return Pair(favorites, nonFavorites)
}
@@ -378,4 +395,3 @@ object SyncManager {
}
}
}

110
server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncYomiSyncService.kt
View File

@@ -31,13 +31,16 @@ object SyncYomiSyncService {
private val network: NetworkHelper by injectLazy()
private val logger = KotlinLogging.logger {}
private class SyncYomiException(message: String?) : Exception(message)
private class SyncYomiException(
message: String?,
) : Exception(message)
suspend fun doSync(syncData: SyncData): Backup? {
try {
val (remoteData, etag) = pullSyncData()
val finalSyncData = if (remoteData != null) {
val finalSyncData =
if (remoteData != null) {
require(etag.isNotEmpty()) { "ETag should never be empty if remote data is not null" }
logger.debug { "Try update remote data with ETag($etag)" }
mergeSyncData(syncData, remoteData)
@@ -67,7 +70,8 @@ object SyncYomiSyncService {
}
val headers = headersBuilder.build()
val downloadRequest = GET(
val downloadRequest =
GET(
url = downloadUrl,
headers = headers,
)
@@ -85,10 +89,12 @@ object SyncYomiSyncService {
}
if (response.isSuccessful) {
val newETag = response.headers["ETag"]
val newETag =
response.headers["ETag"]
?.takeIf { it.isNotEmpty() } ?: throw SyncYomiException("Missing ETag")
val byteArray = response.body.byteStream().use {
val byteArray =
response.body.byteStream().use {
return@use it.readBytes()
}
@@ -108,7 +114,10 @@ object SyncYomiSyncService {
}
}
private suspend fun pushSyncData(syncData: SyncData, eTag: String) {
private suspend fun pushSyncData(
syncData: SyncData,
eTag: String,
) {
val backup = syncData.backup ?: return
val host = serverConfig.syncYomiHost.value
@@ -123,7 +132,9 @@ object SyncYomiSyncService {
val headers = headersBuilder.build()
// Set timeout to 30 seconds
val client = network.client.newBuilder()
val client =
network.client
.newBuilder()
.connectTimeout(timeout, TimeUnit.SECONDS)
.readTimeout(timeout, TimeUnit.SECONDS)
.writeTimeout(timeout, TimeUnit.SECONDS)
@@ -135,7 +146,8 @@ object SyncYomiSyncService {
}
val body = byteArray.toRequestBody("application/octet-stream".toMediaType())
val uploadRequest = PUT(
val uploadRequest =
PUT(
url = uploadUrl,
headers = headers,
body = body,
@@ -144,9 +156,11 @@ object SyncYomiSyncService {
val response = client.newCall(uploadRequest).await()
if (response.isSuccessful) {
val newETag = response.headers["ETag"]
val newETag =
response.headers["ETag"]
?.takeIf { it.isNotEmpty() } ?: throw SyncYomiException("Missing ETag")
syncPreferences.edit()
syncPreferences
.edit()
.putString("last_sync_etag", newETag)
.apply()
logger.debug { "SyncYomi sync completed" }
@@ -159,11 +173,15 @@ object SyncYomiSyncService {
}
}
fun mergeSyncData(localSyncData: SyncData, remoteSyncData: SyncData): SyncData {
fun mergeSyncData(
localSyncData: SyncData,
remoteSyncData: SyncData,
): SyncData {
val mergedCategoriesList =
mergeCategoriesLists(localSyncData.backup?.backupCategories, remoteSyncData.backup?.backupCategories)
val mergedMangaList = mergeMangaLists(
val mergedMangaList =
mergeMangaLists(
localSyncData.backup?.backupManga,
remoteSyncData.backup?.backupManga,
localSyncData.backup?.backupCategories ?: emptyList(),
@@ -175,7 +193,8 @@ object SyncYomiSyncService {
mergeSourcesLists(localSyncData.backup?.backupSources, remoteSyncData.backup?.backupSources)
// Create the merged Backup object
val mergedBackup = Backup(
val mergedBackup =
Backup(
backupManga = mergedMangaList,
backupCategories = mergedCategoriesList,
backupSources = mergedSourcesList,
@@ -201,9 +220,8 @@ object SyncYomiSyncService {
logger.debug { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" }
fun mangaCompositeKey(manga: BackupManga): String {
return "${manga.source}|${manga.url}|${manga.title.lowercase().trim()}|${manga.author?.lowercase()?.trim()}"
}
fun mangaCompositeKey(manga: BackupManga): String =
"${manga.source}|${manga.url}|${manga.title.lowercase().trim()}|${manga.author?.lowercase()?.trim()}"
// Create maps using composite keys
val localMangaMap = localMangaListSafe.associateBy { mangaCompositeKey(it) }
@@ -213,26 +231,36 @@ object SyncYomiSyncService {
val remoteCategoriesMapByOrder = remoteCategories.associateBy { it.order }
val mergedCategoriesMapByName = mergedCategories.associateBy { it.name }
fun updateCategories(theManga: BackupManga, theMap: Map<Int, BackupCategory>): BackupManga {
return theManga.copy(
categories = theManga.categories.mapNotNull {
fun updateCategories(
theManga: BackupManga,
theMap: Map<Int, BackupCategory>,
): BackupManga =
theManga.copy(
categories =
theManga.categories.mapNotNull {
theMap[it]?.let { category ->
mergedCategoriesMapByName[category.name]?.order
}
},
)
}
logger.debug { "Starting merge. Local list size: ${localMangaListSafe.size}, Remote list size: ${remoteMangaListSafe.size}" }
val mergedList = (localMangaMap.keys + remoteMangaMap.keys).distinct().mapNotNull { compositeKey ->
val mergedList =
(localMangaMap.keys + remoteMangaMap.keys).distinct().mapNotNull { compositeKey ->
val local = localMangaMap[compositeKey]
val remote = remoteMangaMap[compositeKey]
// New version comparison logic
when {
local != null && remote == null -> updateCategories(local, localCategoriesMapByOrder)
local == null && remote != null -> updateCategories(remote, remoteCategoriesMapByOrder)
local != null && remote == null -> {
updateCategories(local, localCategoriesMapByOrder)
}
local == null && remote != null -> {
updateCategories(remote, remoteCategoriesMapByOrder)
}
local != null && remote != null -> {
// Compare versions to decide which manga to keep
if (local.version >= remote.version) {
@@ -250,14 +278,18 @@ object SyncYomiSyncService {
}
}
else -> null // No manga found for key
else -> {
null
} // No manga found for key
}
}
// Counting favorites and non-favorites
val (favorites, nonFavorites) = mergedList.partition { it.favorite }
logger.debug { "Merge completed. Total merged manga: ${mergedList.size}, Favorites: ${favorites.size}, Non-Favorites: ${nonFavorites.size}" }
logger.debug {
"Merge completed. Total merged manga: ${mergedList.size}, Favorites: ${favorites.size}, Non-Favorites: ${nonFavorites.size}"
}
return mergedList
}
@@ -266,9 +298,7 @@ object SyncYomiSyncService {
localChapters: List<BackupChapter>,
remoteChapters: List<BackupChapter>,
): List<BackupChapter> {
fun chapterCompositeKey(chapter: BackupChapter): String {
return "${chapter.url}|${chapter.name}|${chapter.chapterNumber}"
}
fun chapterCompositeKey(chapter: BackupChapter): String = "${chapter.url}|${chapter.name}|${chapter.chapterNumber}"
val localChapterMap = localChapters.associateBy { chapterCompositeKey(it) }
val remoteChapterMap = remoteChapters.associateBy { chapterCompositeKey(it) }
@@ -276,11 +306,14 @@ object SyncYomiSyncService {
logger.debug { "Starting chapter merge. Local chapters: ${localChapters.size}, Remote chapters: ${remoteChapters.size}" }
// Merge both chapter maps based on version numbers
val mergedChapters = (localChapterMap.keys + remoteChapterMap.keys).distinct().mapNotNull { compositeKey ->
val mergedChapters =
(localChapterMap.keys + remoteChapterMap.keys).distinct().mapNotNull { compositeKey ->
val localChapter = localChapterMap[compositeKey]
val remoteChapter = remoteChapterMap[compositeKey]
logger.debug { "Processing chapter key: $compositeKey. Local chapter: ${localChapter != null}, Remote chapter: ${remoteChapter != null}" }
logger.debug {
"Processing chapter key: $compositeKey. Local chapter: ${localChapter != null}, Remote chapter: ${remoteChapter != null}"
}
when {
localChapter != null && remoteChapter == null -> {
@@ -295,7 +328,8 @@ object SyncYomiSyncService {
localChapter != null && remoteChapter != null -> {
// Use version number to decide which chapter to keep
val chosenChapter = if (localChapter.version >= remoteChapter.version) {
val chosenChapter =
if (localChapter.version >= remoteChapter.version) {
// If there mare more chapter on remote, local sourceOrder will need to be updated to maintain correct source order.
if (localChapters.size < remoteChapters.size) {
localChapter.copy(sourceOrder = remoteChapter.sourceOrder)
@@ -305,7 +339,9 @@ object SyncYomiSyncService {
} else {
remoteChapter
}
logger.debug { "Merging chapter: ${chosenChapter.name}. Chosen version from: ${if (localChapter.version >= remoteChapter.version) "Local" else "Remote"}, Local version: ${localChapter.version}, Remote version: ${remoteChapter.version}." }
logger.debug {
"Merging chapter: ${chosenChapter.name}. Chosen version from: ${if (localChapter.version >= remoteChapter.version) "Local" else "Remote"}, Local version: ${localChapter.version}, Remote version: ${remoteChapter.version}."
}
chosenChapter
}
@@ -336,7 +372,8 @@ object SyncYomiSyncService {
val remoteCategory = remoteCategoriesMap[name]
if (remoteCategory != null) {
// Compare and merge local and remote categories
val mergedCategory = if (localCategory.order > remoteCategory.order) {
val mergedCategory =
if (localCategory.order > remoteCategory.order) {
localCategory
} else {
remoteCategory
@@ -369,11 +406,14 @@ object SyncYomiSyncService {
logger.debug { "Starting source merge. Local sources: ${localSources?.size}, Remote sources: ${remoteSources?.size}" }
// Merge both source maps
val mergedSources = (localSourceMap.keys + remoteSourceMap.keys).distinct().mapNotNull { sourceId ->
val mergedSources =
(localSourceMap.keys + remoteSourceMap.keys).distinct().mapNotNull { sourceId ->
val localSource = localSourceMap[sourceId]
val remoteSource = remoteSourceMap[sourceId]
logger.debug { "Processing source ID: $sourceId. Local source: ${localSource != null}, Remote source: ${remoteSource != null}" }
logger.debug {
"Processing source ID: $sourceId. Local source: ${localSource != null}, Remote source: ${remoteSource != null}"
}
when {
localSource != null && remoteSource == null -> {

2
server/src/main/kotlin/suwayomi/tachidesk/graphql/types/SyncType.kt
View File

@@ -1,3 +1,5 @@
@file:Suppress("ktlint:standard:filename")
package suwayomi.tachidesk.graphql.types
enum class StartSyncResult {

3
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/ProtoBackupImport.kt
View File

@@ -165,8 +165,7 @@ object ProtoBackupImport : ProtoBackupBase() {
.source()
.run {
if (!isSync) gzip() else this
}
.buffer()
}.buffer()
.use { it.readByteArray() }
val backup = parser.decodeFromByteArray(Backup.serializer(), backupString)

2
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/backup/proto/handlers/BackupMangaHandler.kt
View File

@@ -109,7 +109,7 @@ object BackupMangaHandler {
it.uploadDate,
it.chapterNumber,
chapters.size - it.index,
it.version
it.version,
).apply {
if (flags.includeClientData) {
this.meta = chapterToMeta[it.id] ?: emptyMap()

9
server/src/main/kotlin/suwayomi/tachidesk/server/database/migration/M0053_SyncYomi.kt
View File

@@ -6,12 +6,14 @@ import suwayomi.tachidesk.server.serverConfig
@Suppress("ClassName", "unused")
class M0053_SyncYomi : SQLMigration() {
override val sql = when (serverConfig.databaseType.value) {
override val sql =
when (serverConfig.databaseType.value) {
DatabaseType.POSTGRESQL -> postgresQuery()
DatabaseType.H2 -> h2Query()
}
fun postgresQuery(): String = """
fun postgresQuery(): String =
"""
ALTER TABLE manga ADD COLUMN version BIGINT DEFAULT 0;
ALTER TABLE manga ADD COLUMN is_syncing BOOLEAN DEFAULT FALSE;
@@ -77,7 +79,8 @@ class M0053_SyncYomi : SQLMigration() {
EXECUTE FUNCTION insert_manga_category_update_version();
""".trimIndent()
fun h2Query() = """
fun h2Query() =
"""
ALTER TABLE manga ADD COLUMN version BIGINT DEFAULT 0;
ALTER TABLE manga ADD COLUMN is_syncing BOOLEAN DEFAULT FALSE;

19
server/src/main/kotlin/suwayomi/tachidesk/server/database/trigger/SyncYomiTriggers.kt
View File

@@ -12,14 +12,16 @@ class UpdateMangaVersionTrigger : TriggerAdapter() {
newRow: ResultSet,
) {
val isSyncing = newRow.getBoolean("is_syncing")
val hasChanged = oldRow.getString("url") != newRow.getString("url") ||
val hasChanged =
oldRow.getString("url") != newRow.getString("url") ||
oldRow.getString("description") != newRow.getString("description") ||
oldRow.getBoolean("in_library") != newRow.getBoolean("in_library")
if (!isSyncing && hasChanged) {
val id = newRow.getInt("id")
conn.prepareStatement(
conn
.prepareStatement(
"UPDATE MANGA SET version = version + 1 WHERE id = ?",
).use {
it.setInt(1, id)
@@ -37,7 +39,8 @@ class UpdateChapterAndMangaVersionTrigger : TriggerAdapter() {
newRow: ResultSet,
) {
val isSyncing = newRow.getBoolean("is_syncing")
val hasChanged = oldRow.getBoolean("read") != newRow.getBoolean("read") ||
val hasChanged =
oldRow.getBoolean("read") != newRow.getBoolean("read") ||
oldRow.getBoolean("bookmark") != newRow.getBoolean("bookmark") ||
oldRow.getInt("last_page_read") != newRow.getInt("last_page_read")
@@ -45,14 +48,16 @@ class UpdateChapterAndMangaVersionTrigger : TriggerAdapter() {
val chapterId = newRow.getInt("id")
val mangaId = newRow.getInt("manga")
conn.prepareStatement(
conn
.prepareStatement(
"UPDATE CHAPTER SET version = version + 1 WHERE id = ?",
).use {
it.setInt(1, chapterId)
it.executeUpdate()
}
conn.prepareStatement(
conn
.prepareStatement(
"UPDATE MANGA SET version = version + 1 WHERE id = ? AND (SELECT is_syncing FROM MANGA WHERE id = ?) = FALSE",
).use {
it.setInt(1, mangaId)
@@ -72,7 +77,8 @@ class InsertMangaCategoryUpdateVersionTrigger : TriggerAdapter() {
) {
val mangaId = newRow.getInt("manga")
conn.prepareStatement(
conn
.prepareStatement(
"UPDATE MANGA SET version = version + 1 WHERE id = ? AND (SELECT is_syncing FROM MANGA WHERE id = ?) = FALSE",
).use {
it.setInt(1, mangaId)
@@ -81,4 +87,3 @@ class InsertMangaCategoryUpdateVersionTrigger : TriggerAdapter() {
}
}
}