temple/Suwayomi-Server
1
0
Fork 0
mirror of https://github.com/Suwayomi/Suwayomi-Server.git synced 2026-07-10 22:34:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

Database improvements

Browse Source 
- Move entire sync operation into a single transaction
- Stop loading all manga to memory
This commit is contained in:
Bartu Özen
2025-12-20 23:22:08 +03:00
parent 4558ddf4a6
commit bee8d214c3
1 changed files with 154 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
server/src/main/kotlin/suwayomi/tachidesk/global/impl/sync/SyncManager.kt
View File

@@ -11,8 +11,9 @@ import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import kotlinx.serialization.Serializable
import kotlinx.serialization.protobuf.ProtoBuf
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction
import org.jetbrains.exposed.sql.update
import suwayomi.tachidesk.graphql.types.StartSyncResult
import suwayomi.tachidesk.manga.impl.Category
@@ -122,16 +123,13 @@ object SyncManager {
}
private suspend fun syncData() {
transaction {
newSuspendedTransaction {
MangaTable.update({ MangaTable.isSyncing eq true }) {
it[isSyncing] = false
}
ChapterTable.update({ ChapterTable.isSyncing eq true }) {
it[isSyncing] = false
}
}
val databaseManga = getAllMangaThatNeedsSync()
val backupFlags =
BackupFlags(
@@ -165,7 +163,7 @@ object SyncManager {
if (remoteBackup == null) {
logger.debug { "Skip restore due to network issues" }
// should we call showSyncError?
return
return@newSuspendedTransaction
}
if (remoteBackup === syncData.backup) {
@@ -175,22 +173,28 @@ object SyncManager {
.edit()
.putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
.apply()
return
return@newSuspendedTransaction
}
// Stop the sync early if the remote backup is null or empty
if (remoteBackup.backupManga.isEmpty()) {
return
return@newSuspendedTransaction
}
val isLibraryEmpty =
MangaTable
.selectAll()
.where { MangaTable.inLibrary eq true }
.empty()
// Check if it's first sync based on lastSyncTimestamp
if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && databaseManga.isNotEmpty()) {
if (syncPreferences.getLong("last_sync_timestamp", 0) == 0L && !isLibraryEmpty) {
// It's first sync no need to restore data. (just update remote data)
syncPreferences
.edit()
.putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
.apply()
return
return@newSuspendedTransaction
}
val (filteredFavorites, nonFavorites) = filterFavoritesAndNonFavorites(remoteBackup)
@@ -210,7 +214,7 @@ object SyncManager {
.edit()
.putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
.apply()
return
return@newSuspendedTransaction
}
val backupStream = ProtoBuf.encodeToByteArray(Backup.serializer(), newSyncData).inputStream()
@@ -235,12 +239,6 @@ object SyncManager {
.putLong("last_sync_timestamp", Clock.System.now().toEpochMilliseconds())
.apply()
}
private fun getAllMangaFromDB(): List<MangaDataClass> = transaction { MangaTable.selectAll().map { MangaTable.toDataClass(it) } }
private fun getAllMangaThatNeedsSync(): List<MangaDataClass> =
transaction {
MangaTable.selectAll().where { MangaTable.inLibrary eq true }.map { MangaTable.toDataClass(it) }
}
private fun isMangaDifferent(
@@ -248,20 +246,17 @@ object SyncManager {
remoteManga: BackupManga,
): Boolean {
val localChapters =
transaction {
ChapterTable
.selectAll()
.where { ChapterTable.manga eq localManga.id }
.map { ChapterTable.toDataClass(it) }
}
val localCategories =
transaction {
CategoryMangaTable
.innerJoin(CategoryTable)
.selectAll()
.where { CategoryMangaTable.manga eq localManga.id }
.map { it[CategoryTable.order] }
}
if (areChaptersDifferent(localChapters, remoteManga.chapters)) {
return true
@@ -307,17 +302,20 @@ object SyncManager {
val elapsedTime =
measureTime {
val databaseManga = getAllMangaFromDB()
val localMangaMap =
databaseManga.associateBy {
Triple(it.sourceId.toLong(), it.url, it.title)
}
logger.debug { "Starting to filter favorites and non-favorites from backup data." }
backup.backupManga.forEach { remoteManga ->
val compositeKey = Triple(remoteManga.source, remoteManga.url, remoteManga.title)
val localManga = localMangaMap[compositeKey]
val localManga =
MangaTable
.selectAll()
.where {
(MangaTable.sourceReference eq remoteManga.source) and
(MangaTable.url eq remoteManga.url) and
(MangaTable.title eq remoteManga.title)
}.limit(1)
.map { MangaTable.toDataClass(it) }
.firstOrNull()
when {
// Checks if the manga is in favorites and needs updating or adding
remoteManga.favorite -> {
@@ -346,13 +344,19 @@ object SyncManager {
}
private fun updateNonFavorites(nonFavorites: List<BackupManga>) {
val localMangaList = getAllMangaFromDB()
val localMangaMap = localMangaList.associateBy { Triple(it.sourceId.toLong(), it.url, it.title) }
nonFavorites.forEach { nonFavorite ->
val key = Triple(nonFavorite.source, nonFavorite.url, nonFavorite.title)
localMangaMap[key]?.let { localManga ->
val localManga =
MangaTable
.selectAll()
.where {
(MangaTable.sourceReference eq nonFavorite.source) and
(MangaTable.url eq nonFavorite.url) and
(MangaTable.title eq nonFavorite.title)
}.limit(1)
.map { MangaTable.toDataClass(it) }
.firstOrNull()
if (localManga != null) {
if (localManga.inLibrary != nonFavorite.favorite) {
val updatedManga = localManga.copy(inLibrary = nonFavorite.favorite)
updateManga(updatedManga)
@@ -362,7 +366,6 @@ object SyncManager {
}
private fun updateManga(manga: MangaDataClass) {
transaction {
MangaTable.update({ MangaTable.id eq manga.id }) {
it[MangaTable.url] = manga.url
it[MangaTable.title] = manga.title
@@ -392,4 +395,3 @@ object SyncManager {
}
}
}
}