Feature/updater provide more info about update (#657)

* Provide last global update timestamp

* Provide skipped mangas in update status

* Extract update status logic into function

* Rename update "statusMap" to "mangaStatusMap"

* Provide info about categories in update status

server/src/main/kotlin/suwayomi/tachidesk/graphql/dataLoaders/CategoryDataLoader.kt

@@ -36,6 +36,22 @@ class CategoryDataLoader : KotlinDataLoader<Int, CategoryType> {
     }
 }
 
+class CategoryForIdsDataLoader : KotlinDataLoader<List<Int>, CategoryNodeList> {
+    override val dataLoaderName = "CategoryForIdsDataLoader"
+    override fun getDataLoader(): DataLoader<List<Int>, CategoryNodeList> = DataLoaderFactory.newDataLoader { categoryIds ->
+        future {
+            transaction {
+                addLogger(Slf4jSqlDebugLogger)
+                val ids = categoryIds.flatten().distinct()
+                val categories = CategoryTable.select { CategoryTable.id inList ids }.map { CategoryType(it) }
+                categoryIds.map { categoryIds ->
+                    categories.filter { it.id in categoryIds }.toNodeList()
+                }
+            }
+        }
+    }
+}
+
 class CategoriesForMangaDataLoader : KotlinDataLoader<Int, CategoryNodeList> {
     override val dataLoaderName = "CategoriesForMangaDataLoader"
     override fun getDataLoader(): DataLoader<Int, CategoryNodeList> = DataLoaderFactory.newDataLoader<Int, CategoryNodeList> { ids ->

server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdateQuery.kt

@@ -4,21 +4,18 @@ import org.kodein.di.DI
 import org.kodein.di.conf.global
 import org.kodein.di.instance
 import suwayomi.tachidesk.graphql.types.UpdateStatus
-import suwayomi.tachidesk.graphql.types.UpdateStatusType
 import suwayomi.tachidesk.manga.impl.update.IUpdater
-import suwayomi.tachidesk.manga.impl.update.JobStatus
 
 class UpdateQuery {
     private val updater by DI.global.instance<IUpdater>()
 
     fun updateStatus(): UpdateStatus {
-        val status = updater.status.value
+        return UpdateStatus(updater.status.value)
-        return UpdateStatus(
+    }
-            isRunning = status.running,
+
-            pendingJobs = UpdateStatusType(status.statusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
+    data class LastUpdateTimestampPayload(val timestamp: Long)
-            runningJobs = UpdateStatusType(status.statusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
+
-            completeJobs = UpdateStatusType(status.statusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
+    fun lastUpdateTimestamp(): LastUpdateTimestampPayload {
-            failedJobs = UpdateStatusType(status.statusMap[JobStatus.FAILED]?.map { it.id }.orEmpty())
+        return LastUpdateTimestampPayload(updater.getLastUpdateTimestamp())
-        )
     }
 }

server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskDataLoaderRegistryFactory.kt

@@ -10,6 +10,7 @@ package suwayomi.tachidesk.graphql.server
 import com.expediagroup.graphql.dataloader.KotlinDataLoaderRegistryFactory
 import suwayomi.tachidesk.graphql.dataLoaders.CategoriesForMangaDataLoader
 import suwayomi.tachidesk.graphql.dataLoaders.CategoryDataLoader
+import suwayomi.tachidesk.graphql.dataLoaders.CategoryForIdsDataLoader
 import suwayomi.tachidesk.graphql.dataLoaders.CategoryMetaDataLoader
 import suwayomi.tachidesk.graphql.dataLoaders.ChapterDataLoader
 import suwayomi.tachidesk.graphql.dataLoaders.ChapterMetaDataLoader
@@ -39,6 +40,7 @@ class TachideskDataLoaderRegistryFactory {
                 MangaForSourceDataLoader(),
                 MangaForIdsDataLoader(),
                 CategoryDataLoader(),
+                CategoryForIdsDataLoader(),
                 CategoryMetaDataLoader(),
                 CategoriesForMangaDataLoader(),
                 SourceDataLoader(),

server/src/main/kotlin/suwayomi/tachidesk/graphql/types/UpdateType.kt

@@ -3,26 +3,42 @@ package suwayomi.tachidesk.graphql.types
 import com.expediagroup.graphql.generator.annotations.GraphQLIgnore
 import com.expediagroup.graphql.server.extensions.getValueFromDataLoader
 import graphql.schema.DataFetchingEnvironment
+import suwayomi.tachidesk.manga.impl.update.CategoryUpdateStatus
 import suwayomi.tachidesk.manga.impl.update.JobStatus
 import suwayomi.tachidesk.manga.impl.update.UpdateStatus
 import java.util.concurrent.CompletableFuture
 
 class UpdateStatus(
     val isRunning: Boolean,
+    val skippedCategories: UpdateStatusCategoryType,
+    val updatingCategories: UpdateStatusCategoryType,
     val pendingJobs: UpdateStatusType,
     val runningJobs: UpdateStatusType,
     val completeJobs: UpdateStatusType,
-    val failedJobs: UpdateStatusType
+    val failedJobs: UpdateStatusType,
+    val skippedJobs: UpdateStatusType
 ) {
     constructor(status: UpdateStatus) : this(
         isRunning = status.running,
-        pendingJobs = UpdateStatusType(status.statusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
+        skippedCategories = UpdateStatusCategoryType(status.categoryStatusMap[CategoryUpdateStatus.SKIPPED]?.map { it.id }.orEmpty()),
-        runningJobs = UpdateStatusType(status.statusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
+        updatingCategories = UpdateStatusCategoryType(status.categoryStatusMap[CategoryUpdateStatus.UPDATING]?.map { it.id }.orEmpty()),
-        completeJobs = UpdateStatusType(status.statusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
+        pendingJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.PENDING]?.map { it.id }.orEmpty()),
-        failedJobs = UpdateStatusType(status.statusMap[JobStatus.FAILED]?.map { it.id }.orEmpty())
+        runningJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.RUNNING]?.map { it.id }.orEmpty()),
+        completeJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.COMPLETE]?.map { it.id }.orEmpty()),
+        failedJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.FAILED]?.map { it.id }.orEmpty()),
+        skippedJobs = UpdateStatusType(status.mangaStatusMap[JobStatus.SKIPPED]?.map { it.id }.orEmpty())
     )
 }
 
+class UpdateStatusCategoryType(
+    @get:GraphQLIgnore
+    val categoryIds: List<Int>
+) {
+    fun categories(dataFetchingEnvironment: DataFetchingEnvironment): CompletableFuture<CategoryNodeList> {
+        return dataFetchingEnvironment.getValueFromDataLoader("CategoryForIdsDataLoader", categoryIds)
+    }
+}
+
 class UpdateStatusType(
     @get:GraphQLIgnore
     val mangaIds: List<Int>

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/IUpdater.kt

@@ -4,6 +4,7 @@ import kotlinx.coroutines.flow.StateFlow
 import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
 
 interface IUpdater {
+    fun getLastUpdateTimestamp(): Long
     fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean?, forceAll: Boolean)
     val status: StateFlow<UpdateStatus>
     fun reset()

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateJob.kt

@@ -6,7 +6,8 @@ enum class JobStatus {
     PENDING,
     RUNNING,
     COMPLETE,
-    FAILED
+    FAILED,
+    SKIPPED
 }
 
 data class UpdateJob(

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/UpdateStatus.kt

@@ -1,22 +1,27 @@
 package suwayomi.tachidesk.manga.impl.update
 
 import com.fasterxml.jackson.annotation.JsonIgnore
-import mu.KotlinLogging
+import suwayomi.tachidesk.manga.model.dataclass.CategoryDataClass
 import suwayomi.tachidesk.manga.model.dataclass.MangaDataClass
 
-val logger = KotlinLogging.logger {}
+enum class CategoryUpdateStatus {
+    UPDATING, SKIPPED
+}
+
 data class UpdateStatus(
-    val statusMap: Map<JobStatus, List<MangaDataClass>> = emptyMap(),
+    val categoryStatusMap: Map<CategoryUpdateStatus, List<CategoryDataClass>> = emptyMap(),
+    val mangaStatusMap: Map<JobStatus, List<MangaDataClass>> = emptyMap(),
     val running: Boolean = false,
     @JsonIgnore
     val numberOfJobs: Int = 0
 ) {
 
-    constructor(jobs: List<UpdateJob>, running: Boolean) : this(
+    constructor(categories: Map<CategoryUpdateStatus, List<CategoryDataClass>>, jobs: List<UpdateJob>, skippedMangas: List<MangaDataClass>, running: Boolean) : this(
-        statusMap = jobs.groupBy { it.status }
+        categories,
+        mangaStatusMap = jobs.groupBy { it.status }
             .mapValues { entry ->
                 entry.value.map { it.manga }
-            },
+            }.plus(Pair(JobStatus.SKIPPED, skippedMangas)),
         running = running,
         numberOfJobs = jobs.size
     )

server/src/main/kotlin/suwayomi/tachidesk/manga/impl/update/Updater.kt

@@ -18,9 +18,6 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Semaphore
 import kotlinx.coroutines.sync.withPermit
 import mu.KotlinLogging
-import org.kodein.di.DI
-import org.kodein.di.conf.global
-import org.kodein.di.instance
 import suwayomi.tachidesk.manga.impl.Category
 import suwayomi.tachidesk.manga.impl.CategoryManga
 import suwayomi.tachidesk.manga.impl.Chapter
@@ -49,6 +46,7 @@ class Updater : IUpdater {
     private var maxSourcesInParallel = 20 // max permits, necessary to be set to be able to release up to 20 permits
     private val semaphore = Semaphore(maxSourcesInParallel)
 
+    private val lastUpdateKey = "lastUpdateKey"
     private val lastAutomatedUpdateKey = "lastAutomatedUpdateKey"
     private val preferences = Preferences.userNodeForPackage(Updater::class.java)
 
@@ -76,6 +74,10 @@ class Updater : IUpdater {
         )
     }
 
+    override fun getLastUpdateTimestamp(): Long {
+        return preferences.getLong(lastUpdateKey, 0)
+    }
+
     private fun autoUpdateTask() {
         val lastAutomatedUpdate = preferences.getLong(lastAutomatedUpdateKey, 0)
         preferences.putLong(lastAutomatedUpdateKey, System.currentTimeMillis())
@@ -109,6 +111,15 @@ class Updater : IUpdater {
         HAScheduler.schedule(::autoUpdateTask, updateInterval, timeToNextExecution, "global-update")
     }
 
+    /**
+     * Updates the status and sustains the "skippedMangas"
+     */
+    private fun updateStatus(jobs: List<UpdateJob>, running: Boolean, categories: Map<CategoryUpdateStatus, List<CategoryDataClass>>? = null, skippedMangas: List<MangaDataClass>? = null) {
+        val updateStatusCategories = categories ?: _status.value.categoryStatusMap
+        val tmpSkippedMangas = skippedMangas ?: _status.value.mangaStatusMap[JobStatus.SKIPPED] ?: emptyList()
+        _status.update { UpdateStatus(updateStatusCategories, jobs, tmpSkippedMangas, running) }
+    }
+
     private fun getOrCreateUpdateChannelFor(source: String): Channel<UpdateJob> {
         return updateChannels.getOrPut(source) {
             logger.debug { "getOrCreateUpdateChannelFor: created channel for $source - channels: ${updateChannels.size + 1}" }
@@ -121,7 +132,7 @@ class Updater : IUpdater {
         channel.consumeAsFlow()
             .onEach { job ->
                 semaphore.withPermit {
-                    _status.value = UpdateStatus(
+                    updateStatus(
                         process(job),
                         tracker.any { (_, job) ->
                             job.status == JobStatus.PENDING || job.status == JobStatus.RUNNING
@@ -136,7 +147,7 @@ class Updater : IUpdater {
 
     private suspend fun process(job: UpdateJob): List<UpdateJob> {
         tracker[job.manga.id] = job.copy(status = JobStatus.RUNNING)
-        _status.update { UpdateStatus(tracker.values.toList(), true) }
+        updateStatus(tracker.values.toList(), true)
         tracker[job.manga.id] = try {
             logger.info { "Updating \"${job.manga.title}\" (source: ${job.manga.sourceId})" }
             Chapter.getChapterList(job.manga.id, true)
@@ -150,9 +161,10 @@ class Updater : IUpdater {
     }
 
     override fun addCategoriesToUpdateQueue(categories: List<CategoryDataClass>, clear: Boolean?, forceAll: Boolean) {
-        val updater by DI.global.instance<IUpdater>()
+        preferences.putLong(lastUpdateKey, System.currentTimeMillis())
+
         if (clear == true) {
-            updater.reset()
+            reset()
         }
 
         val includeInUpdateStatusToCategoryMap = categories.groupBy { it.includeInUpdate }
@@ -164,6 +176,11 @@ class Updater : IUpdater {
         } else {
             includedCategories.ifEmpty { unsetCategories }
         }
+        val skippedCategories = categories.subtract(categoriesToUpdate.toSet()).toList()
+        val updateStatusCategories = mapOf(
+            Pair(CategoryUpdateStatus.UPDATING, categoriesToUpdate),
+            Pair(CategoryUpdateStatus.SKIPPED, skippedCategories)
+        )
 
         logger.debug { "Updating categories: '${categoriesToUpdate.joinToString("', '") { it.name }}'" }
 
@@ -179,10 +196,12 @@ class Updater : IUpdater {
             .filter { if (serverConfig.excludeCompleted.value) { it.status != MangaStatus.COMPLETED.name } else true }
             .filter { forceAll || !excludedCategories.any { category -> mangasToCategoriesMap[it.id]?.contains(category) == true } }
             .toList()
+        val skippedMangas = categoriesToUpdateMangas.subtract(mangasToUpdate.toSet()).toList()
 
         // In case no manga gets updated and no update job was running before, the client would never receive an info about its update request
+        updateStatus(emptyList(), mangasToUpdate.isNotEmpty(), updateStatusCategories, skippedMangas)
+
         if (mangasToUpdate.isEmpty()) {
-            UpdaterSocket.notifyAllClients(UpdateStatus())
             return
         }
 
@@ -192,10 +211,10 @@ class Updater : IUpdater {
         )
     }
 
-    private fun addMangasToQueue(mangas: List<MangaDataClass>) {
+    private fun addMangasToQueue(mangasToUpdate: List<MangaDataClass>) {
-        mangas.forEach { tracker[it.id] = UpdateJob(it) }
+        mangasToUpdate.forEach { tracker[it.id] = UpdateJob(it) }
-        _status.update { UpdateStatus(tracker.values.toList(), mangas.isNotEmpty()) }
+        updateStatus(tracker.values.toList(), mangasToUpdate.isNotEmpty())
-        mangas.forEach { addMangaToQueue(it) }
+        mangasToUpdate.forEach { addMangaToQueue(it) }
     }
 
     private fun addMangaToQueue(manga: MangaDataClass) {
@@ -208,7 +227,7 @@ class Updater : IUpdater {
     override fun reset() {
         scope.coroutineContext.cancelChildren()
         tracker.clear()
-        _status.update { UpdateStatus() }
+        updateStatus(emptyList(), false)
         updateChannels.forEach { (_, channel) -> channel.cancel() }
         updateChannels.clear()
     }

server/src/main/kotlin/suwayomi/tachidesk/server/util/WebInterfaceManager.kt

@@ -87,7 +87,8 @@ enum class WebUIChannel {
 }
 
 enum class WebUIFlavor(
-    val uiName: String, val repoUrl: String,
+    val uiName: String,
+    val repoUrl: String,
     val versionMappingUrl: String,
     val latestReleaseInfoUrl: String,
     val baseFileName: String