mirror of
https://github.com/Suwayomi/Suwayomi-Server.git
synced 2026-07-12 07:14:35 -05:00
Implement SyncYomi (#1813)
* Implement SyncYomi
* Add ability to select what to sync
* Properly fix default category bug
* Add periodic sync
* Add PostgreSQL support
* Deschedule previous task
* Check if SyncYomi is enabled in syncData function
* Don't allow multiple syncs at the same time
* Convert SyncYomiSyncService to object
* Make startSync non-suspend
* Return a result from startSync
* Sync before library update
* Improvements
* Use NetworkHelper client
* Lint
* Use measureTime
* Database improvements
- Move entire sync operation into a single transaction
- Stop loading all manga to memory
* Revert "Database improvements"
This reverts commit bee8d214c3.
* Actual database improvements
* Remove runBlocking
* Remove title check
* Update updateNonFavorites function
* Update timeout code
* Improve PostgreSQL query
* Create lastSyncState variable
* Create lastSyncStatus query
* Convert lastSyncState to StateFlow
* Create lastSyncStatusChange subscription
* Replace backupRestoreStatus with backupRestoreId
* Add startDate and endDate
* Add logs for sync start and end
* Handle all errors in syncData
* Change category restore function to match Mihon's behavior
* Fix comment
* Remove duplicate BackupMangaHandler.backup call
* Remove duplicated log
* Rename subscription to syncStatusChanged
* Use same flags for restoring
* Update syncInterval config to use DurationSetting
* Update sync scheduling logic
* Reorder conditions to reduce database calls
* Prevent deleted ghost chapters from reappearing during sync
jobobby04/TachiyomiSY#1575
* Improve sync merging categories
jobobby04/TachiyomiSY#1559
* Make columns not null
* Improve H2 triggers
* Add documentation
This commit is contained in:
Bartu Özen
@@ -28,6 +28,7 @@ import kotlinx.coroutines.flow.update
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.sync.Semaphore
|
||||
import kotlinx.coroutines.sync.withPermit
|
||||
import suwayomi.tachidesk.global.impl.sync.SyncManager
|
||||
import suwayomi.tachidesk.manga.impl.Category
|
||||
import suwayomi.tachidesk.manga.impl.CategoryManga
|
||||
import suwayomi.tachidesk.manga.impl.Chapter
|
||||
@@ -337,90 +338,98 @@ class Updater : IUpdater {
|
||||
clear: Boolean?,
|
||||
forceAll: Boolean,
|
||||
) {
|
||||
saveLastUpdateTimestamp()
|
||||
|
||||
if (clear == true) {
|
||||
reset()
|
||||
}
|
||||
|
||||
val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
|
||||
val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.EXCLUDE].orEmpty()
|
||||
val includedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.INCLUDE].orEmpty()
|
||||
val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.UNSET].orEmpty()
|
||||
val categoriesToUpdate =
|
||||
if (forceAll) {
|
||||
categories
|
||||
} else {
|
||||
includedCategories.ifEmpty { unsetCategories }
|
||||
}
|
||||
val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
|
||||
val updateStatusCategories =
|
||||
mapOf(
|
||||
Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
|
||||
Pair(CategoryUpdateStatus.SKIPPED, skippedCategories),
|
||||
)
|
||||
|
||||
logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
|
||||
|
||||
val categoriesToUpdateMangas =
|
||||
categoriesToUpdate
|
||||
.flatMap { CategoryManga.getCategoryMangaList(it.id) }
|
||||
.distinctBy { it.id }
|
||||
val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
|
||||
val mangasToUpdate =
|
||||
categoriesToUpdateMangas
|
||||
.asSequence()
|
||||
.filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
|
||||
.filter {
|
||||
if (serverConfig.excludeUnreadChapters.value) {
|
||||
(it.unreadCount ?: 0L) == 0L
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}.filter {
|
||||
if (it.initialized && serverConfig.excludeNotStarted.value) {
|
||||
it.lastReadAt != null
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}.filter {
|
||||
if (serverConfig.excludeCompleted.value) {
|
||||
it.status != MangaStatus.COMPLETED.name
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}.filter { forceAll || !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
|
||||
.toList()
|
||||
val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
|
||||
|
||||
this.updateStatusCategories = updateStatusCategories
|
||||
this.updateStatusSkippedMangas = skippedMangas
|
||||
|
||||
if (mangasToUpdate.isEmpty()) {
|
||||
// In case no manga gets updated and no update job was running before, the client would never receive an info
|
||||
// about its update request
|
||||
scope.launch {
|
||||
updateStatus(immediate = true)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
scope.launch {
|
||||
updateStatus(
|
||||
categoryUpdates =
|
||||
updateStatusCategories[CategoryUpdateStatus.UPDATING]
|
||||
?.map {
|
||||
CategoryUpdateJob(it, CategoryUpdateStatus.UPDATING)
|
||||
}.orEmpty(),
|
||||
mangaUpdates = mangasToUpdate.map { UpdateJob(it) },
|
||||
isRunning = true,
|
||||
SyncManager.ensureSync()
|
||||
|
||||
saveLastUpdateTimestamp()
|
||||
|
||||
if (clear == true) {
|
||||
reset()
|
||||
}
|
||||
|
||||
val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
|
||||
val excludedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.EXCLUDE].orEmpty()
|
||||
val includedCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.INCLUDE].orEmpty()
|
||||
val unsetCategories = includeInUpdateStatusToCategoryMap[IncludeOrExclude.UNSET].orEmpty()
|
||||
val categoriesToUpdate =
|
||||
if (forceAll) {
|
||||
categories
|
||||
} else {
|
||||
includedCategories.ifEmpty { unsetCategories }
|
||||
}
|
||||
val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
|
||||
val updateStatusCategories =
|
||||
mapOf(
|
||||
Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
|
||||
Pair(CategoryUpdateStatus.SKIPPED, skippedCategories),
|
||||
)
|
||||
|
||||
logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
|
||||
|
||||
val categoriesToUpdateMangas =
|
||||
categoriesToUpdate
|
||||
.flatMap { CategoryManga.getCategoryMangaList(it.id) }
|
||||
.distinctBy { it.id }
|
||||
val mangasToCategoriesMap = CategoryManga.getMangasCategories(categoriesToUpdateMangas.map { it.id })
|
||||
val mangasToUpdate =
|
||||
categoriesToUpdateMangas
|
||||
.asSequence()
|
||||
.filter { it.updateStrategy == UpdateStrategy.ALWAYS_UPDATE }
|
||||
.filter {
|
||||
if (serverConfig.excludeUnreadChapters.value) {
|
||||
(it.unreadCount ?: 0L) == 0L
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}.filter {
|
||||
if (it.initialized && serverConfig.excludeNotStarted.value) {
|
||||
it.lastReadAt != null
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}.filter {
|
||||
if (serverConfig.excludeCompleted.value) {
|
||||
it.status != MangaStatus.COMPLETED.name
|
||||
} else {
|
||||
true
|
||||
}
|
||||
}.filter {
|
||||
forceAll ||
|
||||
!excludedCategories.any { category ->
|
||||
mangasToCategoriesMap[it.id]?.contains(category) == true
|
||||
}
|
||||
}.toList()
|
||||
val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
|
||||
|
||||
this@Updater.updateStatusCategories = updateStatusCategories
|
||||
this@Updater.updateStatusSkippedMangas = skippedMangas
|
||||
|
||||
if (mangasToUpdate.isEmpty()) {
|
||||
// In case no manga gets updated and no update job was running before, the client would never receive an info
|
||||
// about its update request
|
||||
scope.launch {
|
||||
updateStatus(immediate = true)
|
||||
}
|
||||
return@launch
|
||||
}
|
||||
|
||||
scope.launch {
|
||||
updateStatus(
|
||||
categoryUpdates =
|
||||
updateStatusCategories[CategoryUpdateStatus.UPDATING]
|
||||
?.map {
|
||||
CategoryUpdateJob(it, CategoryUpdateStatus.UPDATING)
|
||||
}.orEmpty(),
|
||||
mangaUpdates = mangasToUpdate.map { UpdateJob(it) },
|
||||
isRunning = true,
|
||||
)
|
||||
}
|
||||
|
||||
addMangasToQueue(
|
||||
mangasToUpdate
|
||||
.sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)),
|
||||
)
|
||||
}
|
||||
|
||||
addMangasToQueue(
|
||||
mangasToUpdate
|
||||
.sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER, MangaDataClass::title)),
|
||||
)
|
||||
}
|
||||
|
||||
override fun addMangasToQueue(mangas: List<MangaDataClass>) {
|
||||
|
||||
Reference in New Issue
Block a user