Emit only updater job changes instead of full status (#1302)

The update subscription emitted the full update status, which, depending on how big the status was, took forever because the graphql subscription does not support data loader batching, causing it to run into the n+1 problem

server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/UpdateMutation.kt

@@ -3,14 +3,11 @@ package suwayomi.tachidesk.graphql.mutations
 import graphql.execution.DataFetcherResult
 import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.withTimeout
-import org.jetbrains.exposed.sql.selectAll
-import org.jetbrains.exposed.sql.transactions.transaction
 import suwayomi.tachidesk.graphql.asDataFetcherResult
+import suwayomi.tachidesk.graphql.types.LibraryUpdateStatus
 import suwayomi.tachidesk.graphql.types.UpdateStatus
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.update.IUpdater
-import suwayomi.tachidesk.manga.model.table.CategoryTable
-import suwayomi.tachidesk.manga.model.table.toDataClass
 import suwayomi.tachidesk.server.JavalinSetup.future
 import uy.kohesive.injekt.injectLazy
 import java.util.concurrent.CompletableFuture
@@ -19,6 +16,38 @@ import kotlin.time.Duration.Companion.seconds
 class UpdateMutation {
     private val updater: IUpdater by injectLazy()
 
+    data class UpdateLibraryInput(
+        val clientMutationId: String? = null,
+        val categories: List<Int>?,
+    )
+
+    data class UpdateLibraryPayload(
+        val clientMutationId: String? = null,
+        val updateStatus: LibraryUpdateStatus,
+    )
+
+    fun updateLibrary(input: UpdateLibraryInput): CompletableFuture<DataFetcherResult<UpdateLibraryPayload?>> {
+        updater.addCategoriesToUpdateQueue(
+            Category.getCategoryList().filter { input.categories?.contains(it.id) ?: true },
+            clear = true,
+            forceAll = !input.categories.isNullOrEmpty(),
+        )
+
+        return future {
+            asDataFetcherResult {
+                UpdateLibraryPayload(
+                    input.clientMutationId,
+                    updateStatus =
+                        withTimeout(30.seconds) {
+                            LibraryUpdateStatus(
+                                updater.updates.first(),
+                            )
+                        },
+                )
+            }
+        }
+    }
+
     data class UpdateLibraryMangaInput(
         val clientMutationId: String? = null,
     )
@@ -29,10 +58,11 @@ class UpdateMutation {
     )
 
     fun updateLibraryManga(input: UpdateLibraryMangaInput): CompletableFuture<DataFetcherResult<UpdateLibraryMangaPayload?>> {
-        updater.addCategoriesToUpdateQueue(
-            Category.getCategoryList(),
-            clear = true,
-            forceAll = false,
+        updateLibrary(
+            UpdateLibraryInput(
+                clientMutationId = input.clientMutationId,
+                categories = null,
+            ),
         )
 
         return future {
@@ -59,13 +89,12 @@ class UpdateMutation {
     )
 
     fun updateCategoryManga(input: UpdateCategoryMangaInput): CompletableFuture<DataFetcherResult<UpdateCategoryMangaPayload?>> {
-        val categories =
-            transaction {
-                CategoryTable.selectAll().where { CategoryTable.id inList input.categories }.map {
-                    CategoryTable.toDataClass(it)
-                }
-            }
-        updater.addCategoriesToUpdateQueue(categories, clear = true, forceAll = true)
+        updateLibrary(
+            UpdateLibraryInput(
+                clientMutationId = input.clientMutationId,
+                categories = input.categories,
+            ),
+        )
 
         return future {
             asDataFetcherResult {

server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt

@@ -1,6 +1,8 @@
 package suwayomi.tachidesk.graphql.queries
 
+import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated
 import kotlinx.coroutines.flow.first
+import suwayomi.tachidesk.graphql.types.LibraryUpdateStatus
 import suwayomi.tachidesk.graphql.types.UpdateStatus
 import suwayomi.tachidesk.manga.impl.update.IUpdater
 import suwayomi.tachidesk.server.JavalinSetup.future
@@ -10,8 +12,11 @@ import java.util.concurrent.CompletableFuture
 class UpdateQuery {
     private val updater: IUpdater by injectLazy()
 
+    @GraphQLDeprecated("Replaced with libraryUpdateStatus", ReplaceWith("libraryUpdateStatus"))
     fun updateStatus(): CompletableFuture<UpdateStatus> = future { UpdateStatus(updater.status.first()) }
 
+    fun libraryUpdateStatus(): CompletableFuture<LibraryUpdateStatus> = future { LibraryUpdateStatus(updater.getStatus()) }
+
     data class LastUpdateTimestampPayload(
         val timestamp: Long,
     )

server/src/main/kotlin/suwayomi/tachidesk/graphql/subscriptions/DownloadSubscription.kt

@@ -16,7 +16,7 @@ import suwayomi.tachidesk.graphql.types.DownloadUpdates
 import suwayomi.tachidesk.manga.impl.download.DownloadManager
 
 class DownloadSubscription {
-    @GraphQLDeprecated("Replaced width downloadStatusChanged", ReplaceWith("downloadStatusChanged(input)"))
+    @GraphQLDeprecated("Replaced with downloadStatusChanged", ReplaceWith("downloadStatusChanged(input)"))
     fun downloadChanged(): Flow<DownloadStatus> =
         DownloadManager.status.map { downloadStatus ->
             DownloadStatus(downloadStatus)

server/src/main/kotlin/suwayomi/tachidesk/graphql/subscriptions/UpdateSubscription.kt

@@ -7,17 +7,68 @@
 
 package suwayomi.tachidesk.graphql.subscriptions
 
+import com.expediagroup.graphql.generator.annotations.GraphQLDeprecated
+import com.expediagroup.graphql.generator.annotations.GraphQLDescription
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.map
 import suwayomi.tachidesk.graphql.types.UpdateStatus
+import suwayomi.tachidesk.graphql.types.UpdaterUpdates
 import suwayomi.tachidesk.manga.impl.update.IUpdater
+import suwayomi.tachidesk.manga.impl.update.UpdateUpdates
 import uy.kohesive.injekt.injectLazy
 
 class UpdateSubscription {
     private val updater: IUpdater by injectLazy()
 
+    @GraphQLDeprecated("Replaced with updates", ReplaceWith("updates(input)"))
     fun updateStatusChanged(): Flow<UpdateStatus> =
         updater.status.map { updateStatus ->
             UpdateStatus(updateStatus)
         }
+
+    data class LibraryUpdateStatusChangedInput(
+        @GraphQLDescription(
+            "Sets a max number of updates that can be contained in a updater update message." +
+                "Everything above this limit will be omitted and the \"updateStatus\" should be re-fetched via the " +
+                "corresponding query. Due to the graphql subscription execution strategy not supporting batching for data loaders, " +
+                "the data loaders run into the n+1 problem, which can cause the server to get unresponsive until the status " +
+                "update has been handled. This is an issue e.g. when starting an update.",
+        )
+        val maxUpdates: Int?,
+    )
+
+    fun libraryUpdateStatusChanged(input: LibraryUpdateStatusChangedInput): Flow<UpdaterUpdates> {
+        val omitUpdates = input.maxUpdates != null
+        val maxUpdates = input.maxUpdates ?: 50
+
+        return updater.updates.map { updates ->
+            val categoryUpdatesCount = updates.categoryUpdates.size
+            val mangaUpdatesCount = updates.mangaUpdates.size
+            val totalUpdatesCount = categoryUpdatesCount + mangaUpdatesCount
+
+            val needToOmitUpdates = omitUpdates && totalUpdatesCount > maxUpdates
+            if (!needToOmitUpdates) {
+                return@map UpdaterUpdates(updates, omittedUpdates = false)
+            }
+
+            val maxUpdatesAfterCategoryUpdates = (maxUpdates - categoryUpdatesCount).coerceAtLeast(0)
+
+            // the graphql subscription execution strategy does not support data loader batching which causes the n+1 problem,
+            // thus, too many updates (e.g. on mass enqueue or dequeue) causes unresponsiveness of the server until the
+            // update has been handled
+            UpdaterUpdates(
+                UpdateUpdates(
+                    updates.isRunning,
+                    updates.categoryUpdates.subList(0, maxUpdates),
+                    updates.mangaUpdates.subList(0, maxUpdatesAfterCategoryUpdates),
+                    updates.totalJobs,
+                    updates.finishedJobs,
+                    updates.skippedCategoriesCount,
+                    updates.skippedMangasCount,
+                    updates.initial,
+                ),
+                omittedUpdates = true,
+            )
+        }
+    }
 }

server/src/main/kotlin/suwayomi/tachidesk/graphql/types/CategoryType.kt

@@ -15,6 +15,7 @@ import suwayomi.tachidesk.graphql.server.primitives.Edge
 import suwayomi.tachidesk.graphql.server.primitives.Node
 import suwayomi.tachidesk.graphql.server.primitives.NodeList
 import suwayomi.tachidesk.graphql.server.primitives.PageInfo
+import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
 import suwayomi.tachidesk.manga.model.dataclass.IncludeOrExclude
 import suwayomi.tachidesk.manga.model.table.CategoryTable
 import java.util.concurrent.CompletableFuture
@@ -36,6 +37,15 @@ class CategoryType(
         IncludeOrExclude.fromValue(row[CategoryTable.includeInDownload]),
     )
 
+    constructor(dataClass: CategoryDataClass) : this(
+        dataClass.id,
+        dataClass.order,
+        dataClass.name,
+        dataClass.default,
+        dataClass.includeInUpdate,
+        dataClass.includeInDownload,
+    )
+
     fun mangas(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture<MangaNodeList> =
         dataFetchingEnvironment.getValueFromDataLoader<Int, MangaNodeList>("MangaForCategoryDataLoader", id)
 

server/src/main/kotlin/suwayomi/tachidesk/graphql/types/UpdateType.kt

@@ -1,11 +1,15 @@
 package suwayomi.tachidesk.graphql.types
 
+import com.expediagroup.graphql.generator.annotations.GraphQLDescription
 import com.expediagroup.graphql.generator.annotations.GraphQLIgnore
 import com.expediagroup.graphql.server.extensions.getValueFromDataLoader
 import graphql.schema.DataFetchingEnvironment
+import suwayomi.tachidesk.manga.impl.update.CategoryUpdateJob
 import suwayomi.tachidesk.manga.impl.update.CategoryUpdateStatus
 import suwayomi.tachidesk.manga.impl.update.JobStatus
+import suwayomi.tachidesk.manga.impl.update.UpdateJob
 import suwayomi.tachidesk.manga.impl.update.UpdateStatus
+import suwayomi.tachidesk.manga.impl.update.UpdateUpdates
 import java.util.concurrent.CompletableFuture
 
 private val jobStatusToMangaIdsToCacheClearedStatus = mutableMapOf<JobStatus, MutableMap<Int, Boolean>>()
@@ -47,14 +51,6 @@ class UpdateStatus(
     )
 }
 
-class UpdateStatusCategoryType(
-    @get:GraphQLIgnore
-    val categoryIds: List<Int>,
-) {
-    fun categories(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture<CategoryNodeList> =
-        dataFetchingEnvironment.getValueFromDataLoader("CategoryForIdsDataLoader", categoryIds)
-}
-
 class UpdateStatusType(
     @get:GraphQLIgnore
     val mangaIds: List<Int>,
@@ -85,6 +81,115 @@ class UpdateStatusType(
             }
         }
 
-        return dataFetchingEnvironment.getValueFromDataLoader<List<Int>, MangaNodeList>("MangaForIdsDataLoader", mangaIds)
+        return dataFetchingEnvironment.getValueFromDataLoader<List<Int>, MangaNodeList>(
+            "MangaForIdsDataLoader",
+            mangaIds,
+        )
     }
 }
+
+class UpdateStatusCategoryType(
+    @get:GraphQLIgnore
+    val categoryIds: List<Int>,
+) {
+    fun categories(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture<CategoryNodeList> =
+        dataFetchingEnvironment.getValueFromDataLoader("CategoryForIdsDataLoader", categoryIds)
+}
+
+class LibraryUpdateStatus(
+    val categoryUpdates: List<CategoryUpdateType>,
+    val mangaUpdates: List<MangaUpdateType>,
+    val jobsInfo: UpdaterJobsInfoType,
+) {
+    constructor(updates: UpdateUpdates) : this(
+        categoryUpdates = updates.categoryUpdates.map(::CategoryUpdateType),
+        mangaUpdates = updates.mangaUpdates.map(::MangaUpdateType),
+        jobsInfo =
+            UpdaterJobsInfoType(
+                isRunning = updates.isRunning,
+                totalJobs = updates.totalJobs,
+                finishedJobs = updates.finishedJobs,
+                skippedCategoriesCount = updates.skippedCategoriesCount,
+                skippedMangasCount = updates.skippedMangasCount,
+            ),
+    )
+}
+
+enum class MangaJobStatus {
+    PENDING,
+    RUNNING,
+    COMPLETE,
+    FAILED,
+    SKIPPED,
+}
+
+enum class CategoryJobStatus {
+    UPDATING,
+    SKIPPED,
+}
+
+class MangaUpdateType(
+    val manga: MangaType,
+    val status: MangaJobStatus,
+) {
+    constructor(job: UpdateJob) : this(
+        MangaType(job.manga),
+        when (job.status) {
+            JobStatus.PENDING -> MangaJobStatus.PENDING
+            JobStatus.RUNNING -> MangaJobStatus.RUNNING
+            JobStatus.COMPLETE -> MangaJobStatus.COMPLETE
+            JobStatus.FAILED -> MangaJobStatus.FAILED
+            JobStatus.SKIPPED -> MangaJobStatus.SKIPPED
+        },
+    )
+}
+
+class CategoryUpdateType(
+    val category: CategoryType,
+    val status: CategoryJobStatus,
+) {
+    constructor(job: CategoryUpdateJob) : this(
+        CategoryType(job.category),
+        when (job.status) {
+            CategoryUpdateStatus.UPDATING -> CategoryJobStatus.UPDATING
+            CategoryUpdateStatus.SKIPPED -> CategoryJobStatus.SKIPPED
+        },
+    )
+}
+
+// wrap this info in a data class so that the update subscription updates the date of the update status in the clients cache
+data class UpdaterJobsInfoType(
+    val isRunning: Boolean,
+    val totalJobs: Int,
+    val finishedJobs: Int,
+    val skippedCategoriesCount: Int,
+    val skippedMangasCount: Int,
+)
+
+data class UpdaterUpdates(
+    val categoryUpdates: List<CategoryUpdateType>,
+    val mangaUpdates: List<MangaUpdateType>,
+    @GraphQLDescription("The current update status at the time of sending the initial message. Is null for all following messages")
+    val initial: LibraryUpdateStatus?,
+    val jobsInfo: UpdaterJobsInfoType,
+    @GraphQLDescription(
+        "Indicates whether updates have been omitted based on the \"maxUpdates\" subscription variable. " +
+            "In case updates have been omitted, the \"updateStatus\" query should be re-fetched.",
+    )
+    val omittedUpdates: Boolean,
+) {
+    constructor(updates: UpdateUpdates, omittedUpdates: Boolean) : this(
+        categoryUpdates = updates.categoryUpdates.map(::CategoryUpdateType),
+        mangaUpdates = updates.mangaUpdates.map(::MangaUpdateType),
+        initial = updates.initial?.let { LibraryUpdateStatus(updates.initial) },
+        jobsInfo =
+            UpdaterJobsInfoType(
+                isRunning = updates.isRunning,
+                totalJobs = updates.totalJobs,
+                finishedJobs = updates.finishedJobs,
+                skippedCategoriesCount = updates.skippedCategoriesCount,
+                skippedMangasCount = updates.skippedMangasCount,
+            ),
+        omittedUpdates = omittedUpdates,
+    )
+}