temple/Suwayomi-Server
1
0
Fork 0
mirror of https://github.com/Suwayomi/Suwayomi-Server.git synced 2026-07-11 14:54:33 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix/update subscription clear data loader cache (#908)

Browse Source 
* Set updater running flag to false only at the end of the update

For clearing the data loader cache properly, the update status subscription requires the update to be running.

For the last completed manga update the flag was immediately set to false which prevented the dataloader cache from getting cleared, returning outdated data for the last updated manga

* Correctly clear the "MangaForIdsDataLoader" cache

The cache keys for this dataloader are lists of manga ids.
Thus, it is not possible to clear only the cached data of the provided manga id and instead each cache entry that includes the manga id has to be cleared

* Ensure that manga dataloader caches gets cleared during global update

The "StateFlow" drops value updates in case the collector is too slow, which was the case for the "UpdateSubscription".

This caused the dataloader cache to not get properly cleared because the running state of the update was already set to false.
This commit is contained in:
schroda
2024-03-31 19:19:49 +02:00
committed by GitHub
parent b2aff1efc9
commit 6d539d3404
9 changed files with 190 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/main/kotlin/suwayomi/tachidesk/manga/controller/UpdateController.kt
View File

@@ -117,7 +117,7 @@ object UpdateController {
},
behaviorOf = { ctx ->
val updater by DI.global.instance<IUpdater>()
ctx.json(updater.status.value)
ctx.json(updater.statusDeprecated.value)
},
withResults = {
json<UpdateStatus>(HttpCode.OK)

5
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt
View File

@@ -1,5 +1,6 @@
package suwayomi.tachidesk.manga.impl.update
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.StateFlow
import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
@@ -12,7 +13,9 @@ interface IUpdater {
forceAll: Boolean,
)
val status: StateFlow<UpdateStatus>
val status: Flow<UpdateStatus>
val statusDeprecated: StateFlow<UpdateStatus>
fun reset()
}

103
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt
View File

@@ -6,9 +6,12 @@ import eu.kanade.tachiyomi.source.model.UpdateStrategy
import kotlinx.coroutines.CancellationException
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.FlowPreview
import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.cancelChildren
import kotlinx.coroutines.channels.BufferOverflow
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.flow.MutableSharedFlow
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.asStateFlow
import kotlinx.coroutines.flow.catch
@@ -16,6 +19,8 @@ import kotlinx.coroutines.flow.consumeAsFlow
import kotlinx.coroutines.flow.launchIn
import kotlinx.coroutines.flow.onCompletion
import kotlinx.coroutines.flow.onEach
import kotlinx.coroutines.flow.onStart
import kotlinx.coroutines.flow.sample
import kotlinx.coroutines.flow.update
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Semaphore
@@ -37,14 +42,33 @@ import java.util.Date
import java.util.concurrent.ConcurrentHashMap
import kotlin.math.absoluteValue
import kotlin.time.Duration.Companion.hours
import kotlin.time.Duration.Companion.seconds
@OptIn(FlowPreview::class)
class Updater : IUpdater {
private val logger = KotlinLogging.logger {}
private val notifyFlowScope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
private val _status = MutableStateFlow(UpdateStatus())
override val status = _status.asStateFlow()
private val notifyFlow = MutableSharedFlow<Unit>(extraBufferCapacity = 1, onBufferOverflow = BufferOverflow.DROP_OLDEST)
private val statusFlow = MutableSharedFlow<UpdateStatus>()
override val status = statusFlow.onStart { emit(getStatus()) }
init {
// has to be in its own scope (notifyFlowScope), otherwise, the collection gets canceled due to canceling the scopes (scope) children in the reset function
notifyFlowScope.launch {
notifyFlow.sample(1.seconds).collect {
updateStatus(immediate = true)
}
}
}
private val _status = MutableStateFlow(UpdateStatus())
override val statusDeprecated = _status.asStateFlow()
private var updateStatusCategories: Map<CategoryUpdateStatus, List<CategoryDataClass>> = emptyMap()
private var updateStatusSkippedMangas: List<MangaDataClass> = emptyList()
private val tracker = ConcurrentHashMap<Int, UpdateJob>()
private val updateChannels = ConcurrentHashMap<String, Channel<UpdateJob>>()
@@ -87,7 +111,7 @@ class Updater : IUpdater {
val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
preferences.edit().putLong(lastAutomatedUpdateKey, System.currentTimeMillis()).apply()
if (status.value.running) {
if (getStatus().running) {
logger.debug { "Global update is already in progress" }
return
}
@@ -123,23 +147,33 @@ class Updater : IUpdater {
HAScheduler.schedule(::autoUpdateTask, updateInterval, timeToNextExecution, "global-update")
}
/**
* Updates the status and sustains the "skippedMangas"
*/
private fun updateStatus(
jobs: List<UpdateJob>,
running: Boolean? = null,
categories: Map<CategoryUpdateStatus, List<CategoryDataClass>>? = null,
skippedMangas: List<MangaDataClass>? = null,
) {
private fun getStatus(running: Boolean? = null): UpdateStatus {
val jobs = tracker.values.toList()
val isRunning =
running
?: jobs.any { job ->
job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING
}
val updateStatusCategories = categories ?: _status.value.categoryStatusMap
val tmpSkippedMangas = skippedMangas ?: _status.value.mangaStatusMap[JobStatus.SKIPPED] ?: emptyList()
_status.update { UpdateStatus(updateStatusCategories, jobs, tmpSkippedMangas, isRunning) }
return UpdateStatus(this.updateStatusCategories, jobs, this.updateStatusSkippedMangas, isRunning)
}
/**
* Pass "isRunning" to force a specific running state
*/
private suspend fun updateStatus(
immediate: Boolean = false,
isRunning: Boolean? = null,
) {
if (immediate) {
val status = getStatus(running = isRunning)
statusFlow.emit(status)
_status.update { status }
return
}
notifyFlow.emit(Unit)
}
private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {
@@ -166,7 +200,7 @@ class Updater : IUpdater {
return channel
}
private fun handleChannelUpdateFailure(source: String) {
private suspend fun handleChannelUpdateFailure(source: String) {
val isFailedSourceUpdate = { job: UpdateJob ->
val isForSource = job.manga.sourceId == source
val hasFailed = job.status == JobStatus.FAILED
@@ -181,17 +215,12 @@ class Updater : IUpdater {
tracker[mangaId] = job.copy(status = JobStatus.FAILED)
}
updateStatus(
tracker.values.toList(),
tracker.any { (_, job) ->
job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING
},
)
updateStatus()
}
private suspend fun process(job: UpdateJob) {
tracker[job.manga.id] = job.copy(status = JobStatus.RUNNING)
updateStatus(tracker.values.toList(), true)
updateStatus()
tracker[job.manga.id] =
try {
@@ -207,7 +236,15 @@ class Updater : IUpdater {
job.copy(status = JobStatus.FAILED)
}
updateStatus(tracker.values.toList())
val wasLastJob = tracker.values.none { it.status == JobStatus.PENDING || it.status == JobStatus.RUNNING }
// in case this is the last update job, the running flag has to be true, before it gets set to false, to be able
// to properly clear the dataloader store in UpdateType
updateStatus(immediate = wasLastJob, isRunning = true)
if (wasLastJob) {
updateStatus(isRunning = false)
}
}
override fun addCategoriesToUpdateQueue(
@@ -274,10 +311,15 @@ class Updater : IUpdater {
.toList()
val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
// In case no manga gets updated and no update job was running before, the client would never receive an info about its update request
updateStatus(emptyList(), mangasToUpdate.isNotEmpty(), updateStatusCategories, skippedMangas)
this.updateStatusCategories = updateStatusCategories
this.updateStatusSkippedMangas = skippedMangas
if (mangasToUpdate.isEmpty()) {
// In case no manga gets updated and no update job was running before, the client would never receive an info
// about its update request
scope.launch {
updateStatus(immediate = true)
}
return
}
@@ -288,8 +330,9 @@ class Updater : IUpdater {
}
private fun addMangasToQueue(mangasToUpdate: List<MangaDataClass>) {
// create all manga update jobs before adding them to the queue so that the client is able to calculate the
// progress properly right form the start
mangasToUpdate.forEach { tracker[it.id] = UpdateJob(it) }
updateStatus(tracker.values.toList(), mangasToUpdate.isNotEmpty())
mangasToUpdate.forEach { addMangaToQueue(it) }
}
@@ -303,7 +346,11 @@ class Updater : IUpdater {
override fun reset() {
scope.coroutineContext.cancelChildren()
tracker.clear()
updateStatus(emptyList(), false)
this.updateStatusCategories = emptyMap()
this.updateStatusSkippedMangas = emptyList()
scope.launch {
updateStatus(immediate = true, isRunning = false)
}
updateChannels.forEach { (_, channel) -> channel.cancel() }
updateChannels.clear()
}

4
server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdaterSocket.kt
View File

@@ -23,12 +23,12 @@ object UpdaterSocket : Websocket<UpdateStatus>() {
ctx: WsContext,
value: UpdateStatus?,
) {
ctx.send(value ?: updater.status.value)
ctx.send(value ?: updater.statusDeprecated.value)
}
override fun handleRequest(ctx: WsMessageContext) {
when (ctx.message()) {
"STATUS" -> notifyClient(ctx, updater.status.value)
"STATUS" -> notifyClient(ctx, updater.statusDeprecated.value)
else ->
ctx.send(
"""