Fix/update subscription clear data loader cache (#908)

* Set updater running flag to false only at the end of the update For clearing the data loader cache properly, the update status subscription requires the update to be running. For the last completed manga update the flag was immediately set to false which prevented the dataloader cache from getting cleared, returning outdated data for the last updated manga * Correctly clear the "MangaForIdsDataLoader" cache The cache keys for this dataloader are lists of manga ids. Thus, it is not possible to clear only the cached data of the provided manga id and instead each cache entry that includes the manga id has to be cleared * Ensure that manga dataloader caches gets cleared during global update The "StateFlow" drops value updates in case the collector is too slow, which was the case for the "UpdateSubscription". This caused the dataloader cache to not get properly cleared because the running state of the update was already set to false.
2026-07-03 19:04:39 -05:00 · 2024-03-31 19:19:49 +02:00
parent b2aff1efc9
commit 6d539d3404
9 changed files with 190 additions and 54 deletions
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/cache/CustomCacheMap.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/cache/CustomCacheMap.kt
@@ -0,0 +1,46 @@
+package suwayomi.tachidesk.graphql.cache
+
+import org.dataloader.CacheMap
+import java.util.concurrent.CompletableFuture
+
+class CustomCacheMap<K, V> : CacheMap<K, V> {
+    private val cache: MutableMap<K, CompletableFuture<V>>
+
+    init {
+        cache = HashMap()
+    }
+
+    override fun containsKey(key: K): Boolean {
+        return cache.containsKey(key)
+    }
+
+    override fun get(key: K): CompletableFuture<V> {
+        return cache[key]!!
+    }
+
+    fun getKeys(): Collection<K> {
+        return cache.keys.toSet()
+    }
+
+    override fun getAll(): Collection<CompletableFuture<V>> {
+        return cache.values
+    }
+
+    override fun set(
+        key: K,
+        value: CompletableFuture<V>,
+    ): CacheMap<K, V> {
+        cache[key] = value
+        return this
+    }
+
+    override fun delete(key: K): CacheMap<K, V> {
+        cache.remove(key)
+        return this
+    }
+
+    override fun clear(): CacheMap<K, V> {
+        cache.clear()
+        return this
+    }
+}
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/MangaDataLoader.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/MangaDataLoader.kt
@@ -10,11 +10,13 @@ package suwayomi.tachidesk.graphql.dataLoaders
 import com.expediagroup.graphql.dataloader.KotlinDataLoader
 import org.dataloader.DataLoader
 import org.dataloader.DataLoaderFactory
+import org.dataloader.DataLoaderOptions
 import org.jetbrains.exposed.sql.Slf4jSqlDebugLogger
 import org.jetbrains.exposed.sql.addLogger
 import org.jetbrains.exposed.sql.andWhere
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
+import suwayomi.tachidesk.graphql.cache.CustomCacheMap
 import suwayomi.tachidesk.graphql.types.MangaNodeList
 import suwayomi.tachidesk.graphql.types.MangaNodeList.Companion.toNodeList
 import suwayomi.tachidesk.graphql.types.MangaType
@@ -95,18 +97,21 @@ class MangaForIdsDataLoader : KotlinDataLoader<List<Int>, MangaNodeList> {
    override val dataLoaderName = "MangaForIdsDataLoader"

    override fun getDataLoader(): DataLoader<List<Int>, MangaNodeList> =
-        DataLoaderFactory.newDataLoader { mangaIds ->
-            future {
-                transaction {
-                    addLogger(Slf4jSqlDebugLogger)
-                    val ids = mangaIds.flatten().distinct()
-                    val manga =
-                        MangaTable.select { MangaTable.id inList ids }
-                            .map { MangaType(it) }
-                    mangaIds.map { mangaIds ->
-                        manga.filter { it.id in mangaIds }.toNodeList()
+        DataLoaderFactory.newDataLoader(
+            { mangaIds ->
+                future {
+                    transaction {
+                        addLogger(Slf4jSqlDebugLogger)
+                        val ids = mangaIds.flatten().distinct()
+                        val manga =
+                            MangaTable.select { MangaTable.id inList ids }
+                                .map { MangaType(it) }
+                        mangaIds.map { mangaIds ->
+                            manga.filter { it.id in mangaIds }.toNodeList()
+                        }
                    }
                }
-            }
-        }
+            },
+            DataLoaderOptions.newOptions().setCacheMap(CustomCacheMap<List<Int>, MangaNodeList>()),
+        )
 }
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/UpdateMutation.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/UpdateMutation.kt
@@ -1,5 +1,7 @@
 package suwayomi.tachidesk.graphql.mutations

+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.withTimeout
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.kodein.di.DI
@@ -10,6 +12,9 @@ import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.update.IUpdater
 import suwayomi.tachidesk.manga.model.table.CategoryTable
 import suwayomi.tachidesk.manga.model.table.toDataClass
+import suwayomi.tachidesk.server.JavalinSetup.future
+import java.util.concurrent.CompletableFuture
+import kotlin.time.Duration.Companion.seconds

 class UpdateMutation {
    private val updater by DI.global.instance<IUpdater>()
@@ -23,14 +28,22 @@ class UpdateMutation {
        val updateStatus: UpdateStatus,
    )

-    fun updateLibraryManga(input: UpdateLibraryMangaInput): UpdateLibraryMangaPayload {
+    fun updateLibraryManga(input: UpdateLibraryMangaInput): CompletableFuture<UpdateLibraryMangaPayload> {
        updater.addCategoriesToUpdateQueue(
            Category.getCategoryList(),
            clear = true,
            forceAll = false,
        )

-        return UpdateLibraryMangaPayload(input.clientMutationId, UpdateStatus(updater.status.value))
+        return future {
+            UpdateLibraryMangaPayload(
+                input.clientMutationId,
+                updateStatus =
+                    withTimeout(30.seconds) {
+                        UpdateStatus(updater.status.first())
+                    },
+            )
+        }
    }

    data class UpdateCategoryMangaInput(
@@ -43,7 +56,7 @@ class UpdateMutation {
        val updateStatus: UpdateStatus,
    )

-    fun updateCategoryManga(input: UpdateCategoryMangaInput): UpdateCategoryMangaPayload {
+    fun updateCategoryManga(input: UpdateCategoryMangaInput): CompletableFuture<UpdateCategoryMangaPayload> {
        val categories =
            transaction {
                CategoryTable.select { CategoryTable.id inList input.categories }.map {
@@ -52,10 +65,15 @@ class UpdateMutation {
            }
        updater.addCategoriesToUpdateQueue(categories, clear = true, forceAll = true)

-        return UpdateCategoryMangaPayload(
-            clientMutationId = input.clientMutationId,
-            updateStatus = UpdateStatus(updater.status.value),
-        )
+        return future {
+            UpdateCategoryMangaPayload(
+                input.clientMutationId,
+                updateStatus =
+                    withTimeout(30.seconds) {
+                        UpdateStatus(updater.status.first())
+                    },
+            )
+        }
    }

    data class UpdateStopInput(
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt
@@ -1,16 +1,19 @@
 package suwayomi.tachidesk.graphql.queries

+import kotlinx.coroutines.flow.first
 import org.kodein.di.DI
 import org.kodein.di.conf.global
 import org.kodein.di.instance
 import suwayomi.tachidesk.graphql.types.UpdateStatus
 import suwayomi.tachidesk.manga.impl.update.IUpdater
+import suwayomi.tachidesk.server.JavalinSetup.future
+import java.util.concurrent.CompletableFuture

 class UpdateQuery {
    private val updater by DI.global.instance<IUpdater>()

-    fun updateStatus(): UpdateStatus {
-        return UpdateStatus(updater.status.value)
+    fun updateStatus(): CompletableFuture<UpdateStatus> {
+        return future { UpdateStatus(updater.status.first()) }
    }

    data class LastUpdateTimestampPayload(val timestamp: Long)
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/MangaType.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/MangaType.kt
@@ -11,6 +11,7 @@ import com.expediagroup.graphql.server.extensions.getValueFromDataLoader
 import eu.kanade.tachiyomi.source.model.UpdateStrategy
 import graphql.schema.DataFetchingEnvironment
 import org.jetbrains.exposed.sql.ResultRow
+import suwayomi.tachidesk.graphql.cache.CustomCacheMap
 import suwayomi.tachidesk.graphql.server.primitives.Cursor
 import suwayomi.tachidesk.graphql.server.primitives.Edge
 import suwayomi.tachidesk.graphql.server.primitives.Node
@@ -45,12 +46,25 @@ class MangaType(
    var chaptersLastFetchedAt: Long?, // todo
 ) : Node {
    companion object {
+        fun clearCacheFor(
+            mangaIds: List<Int>,
+            dataFetchingEnvironment: DataFetchingEnvironment,
+        ) {
+            mangaIds.forEach { clearCacheFor(it, dataFetchingEnvironment) }
+        }
+
        fun clearCacheFor(
            mangaId: Int,
            dataFetchingEnvironment: DataFetchingEnvironment,
        ) {
            dataFetchingEnvironment.getDataLoader<Int, MangaType>("MangaDataLoader").clear(mangaId)
-            dataFetchingEnvironment.getDataLoader<Int, MangaNodeList>("MangaForIdsDataLoader").clear(mangaId)
+
+            val mangaForIdsDataLoader =
+                dataFetchingEnvironment.getDataLoader<List<Int>, MangaNodeList>("MangaForIdsDataLoader")
+            @Suppress("UNCHECKED_CAST")
+            (mangaForIdsDataLoader.cacheMap as CustomCacheMap<List<Int>, MangaNodeList>).getKeys()
+                .filter { it.contains(mangaId) }.forEach { mangaForIdsDataLoader.clear(it) }
+
            dataFetchingEnvironment.getDataLoader<Int, Int>("DownloadedChapterCountForMangaDataLoader").clear(mangaId)
            dataFetchingEnvironment.getDataLoader<Int, Int>("UnreadChapterCountForMangaDataLoader").clear(mangaId)
            dataFetchingEnvironment.getDataLoader<Int, ChapterType>("LastReadChapterForMangaDataLoader").clear(mangaId)