Backup creation and restore gql endpoints (#587)

2026-07-05 11:54:38 -05:00 · 2023-07-02 11:45:44 -04:00
parent 1a9a0b3394
commit a11b654c3d
13 changed files with 477 additions and 43 deletions
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/GraphQL.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/GraphQL.kt
@@ -8,6 +8,7 @@
 package suwayomi.tachidesk.graphql

 import io.javalin.apibuilder.ApiBuilder.get
+import io.javalin.apibuilder.ApiBuilder.path
 import io.javalin.apibuilder.ApiBuilder.post
 import io.javalin.apibuilder.ApiBuilder.ws
 import suwayomi.tachidesk.graphql.controller.GraphQLController
@@ -19,5 +20,9 @@ object GraphQL {

        // graphql playground
        get("graphql", GraphQLController::playground)
+
+        path("graphql/files") {
+            get("backup/{file}", GraphQLController::retrieveFile)
+        }
    }
 }
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/controller/GraphQLController.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/controller/GraphQLController.kt
@@ -11,7 +11,9 @@ import io.javalin.http.ContentType
 import io.javalin.http.Context
 import io.javalin.websocket.WsConfig
 import suwayomi.tachidesk.graphql.server.TachideskGraphQLServer
+import suwayomi.tachidesk.graphql.server.TemporaryFileStorage
 import suwayomi.tachidesk.server.JavalinSetup.future
+import kotlin.io.path.inputStream

 object GraphQLController {
    private val server = TachideskGraphQLServer.create()
@@ -30,6 +32,14 @@ object GraphQLController {
        ctx.result(javaClass.getResourceAsStream("/graphql-playground.html")!!)
    }

+    fun retrieveFile(ctx: Context) {
+        val filename = ctx.pathParam("file")
+        val file = TemporaryFileStorage.retrieveFile(filename)
+        ctx.contentType("application/octet-stream")
+        ctx.header("Content-Disposition", """attachment; filename="$filename"""")
+        ctx.result(file.inputStream())
+    }
+
    fun webSocket(ws: WsConfig) {
        ws.onMessage { ctx ->
            server.handleSubscriptionMessage(ctx)
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/BackupMutation.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/mutations/BackupMutation.kt
@@ -0,0 +1,85 @@
+package suwayomi.tachidesk.graphql.mutations
+
+import io.javalin.http.UploadedFile
+import kotlinx.coroutines.DelicateCoroutinesApi
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withTimeout
+import suwayomi.tachidesk.graphql.server.TemporaryFileStorage
+import suwayomi.tachidesk.graphql.types.BackupRestoreState
+import suwayomi.tachidesk.graphql.types.BackupRestoreStatus
+import suwayomi.tachidesk.graphql.types.toStatus
+import suwayomi.tachidesk.manga.impl.backup.BackupFlags
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupExport
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport
+import suwayomi.tachidesk.server.JavalinSetup.future
+import java.text.SimpleDateFormat
+import java.util.Date
+import java.util.concurrent.CompletableFuture
+import kotlin.time.Duration.Companion.seconds
+
+class BackupMutation {
+    data class RestoreBackupInput(
+        val clientMutationId: String? = null,
+        val backup: UploadedFile
+    )
+    data class RestoreBackupPayload(
+        val clientMutationId: String?,
+        val status: BackupRestoreStatus
+    )
+
+    @OptIn(DelicateCoroutinesApi::class)
+    fun restoreBackup(
+        input: RestoreBackupInput
+    ): CompletableFuture<RestoreBackupPayload> {
+        val (clientMutationId, backup) = input
+
+        return future {
+            GlobalScope.launch {
+                ProtoBackupImport.performRestore(backup.content)
+            }
+
+            val status = withTimeout(10.seconds) {
+                ProtoBackupImport.backupRestoreState.first {
+                    it != ProtoBackupImport.BackupRestoreState.Idle
+                }.toStatus()
+            }
+
+            RestoreBackupPayload(clientMutationId, status)
+        }
+    }
+
+    data class CreateBackupInput(
+        val clientMutationId: String? = null,
+        val includeChapters: Boolean? = null,
+        val includeCategories: Boolean? = null
+    )
+    data class CreateBackupPayload(
+        val clientMutationId: String?,
+        val url: String
+    )
+    fun createBackup(
+        input: CreateBackupInput? = null
+    ): CreateBackupPayload {
+        val currentDate = SimpleDateFormat("yyyy-MM-dd_HH-mm").format(Date())
+        val filename = "tachidesk_$currentDate.proto.gz"
+
+        val backup = ProtoBackupExport.createBackup(
+            BackupFlags(
+                includeManga = true,
+                includeCategories = input?.includeCategories ?: true,
+                includeChapters = input?.includeChapters ?: true,
+                includeTracking = true,
+                includeHistory = true
+            )
+        )
+
+        TemporaryFileStorage.saveFile(filename, backup)
+
+        return CreateBackupPayload(
+            clientMutationId = input?.clientMutationId,
+            url = "/api/graphql/files/backup/$filename"
+        )
+    }
+}
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/BackupQuery.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/BackupQuery.kt
@@ -0,0 +1,32 @@
+package suwayomi.tachidesk.graphql.queries
+
+import io.javalin.http.UploadedFile
+import suwayomi.tachidesk.graphql.types.BackupRestoreStatus
+import suwayomi.tachidesk.graphql.types.toStatus
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupValidator
+
+class BackupQuery {
+    data class ValidateBackupInput(
+        val backup: UploadedFile
+    )
+    data class ValidateBackupSource(
+        val id: Long,
+        val name: String
+    )
+    data class ValidateBackupResult(
+        val missingSources: List<ValidateBackupSource>
+    )
+    fun validateBackup(
+        input: ValidateBackupInput
+    ): ValidateBackupResult {
+        val result = ProtoBackupValidator.validate(input.backup.content)
+        return ValidateBackupResult(
+            result.missingSourceIds.map { ValidateBackupSource(it.first, it.second) }
+        )
+    }
+
+    fun restoreStatus(): BackupRestoreStatus {
+        return ProtoBackupImport.backupRestoreState.value.toStatus()
+    }
+}
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdaterQuery.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/queries/UpdaterQuery.kt
@@ -0,0 +1,42 @@
+package suwayomi.tachidesk.graphql.queries
+
+import org.jetbrains.exposed.sql.select
+import org.jetbrains.exposed.sql.transactions.transaction
+import org.kodein.di.DI
+import org.kodein.di.conf.global
+import org.kodein.di.instance
+import suwayomi.tachidesk.graphql.types.MangaType
+import suwayomi.tachidesk.manga.impl.update.IUpdater
+import suwayomi.tachidesk.manga.impl.update.JobStatus
+import suwayomi.tachidesk.manga.impl.update.UpdaterSocket
+import suwayomi.tachidesk.manga.model.table.MangaTable
+
+class UpdaterQuery {
+    sealed interface UpdaterStatus {
+        data class UpdaterJob(val status: JobStatus, val manga: MangaType)
+
+        data class Running(val jobs: Map<JobStatus, List<MangaType>>) : UpdaterStatus
+
+        // data class Idle
+    }
+
+    private val updater by DI.global.instance<IUpdater>()
+
+    fun updaterStatus() {
+        val status = updater.status.value
+        if (status.running) {
+            val mangaIds = status.statusMap.values.flatMap { mangas -> mangas.map { it.id } }
+            val mangaMap = transaction {
+                MangaTable.select { MangaTable.id inList mangaIds }
+                    .map { MangaType(it) }
+                    .associateBy { it.id }
+            }
+            UpdaterStatus.Running(
+                status.statusMap.mapValues { (_, mangas) ->
+                    mangas.mapNotNull { mangaMap[it.id] }
+                }
+            )
+        }
+        UpdaterSocket
+    }
+}
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/JavalinGraphQLRequestParser.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/JavalinGraphQLRequestParser.kt
@@ -8,16 +8,102 @@
 package suwayomi.tachidesk.graphql.server

 import com.expediagroup.graphql.server.execution.GraphQLRequestParser
+import com.expediagroup.graphql.server.types.GraphQLBatchRequest
+import com.expediagroup.graphql.server.types.GraphQLRequest
 import com.expediagroup.graphql.server.types.GraphQLServerRequest
 import io.javalin.http.Context
+import io.javalin.http.UploadedFile
+import io.javalin.plugin.json.jsonMapper
 import java.io.IOException

 class JavalinGraphQLRequestParser : GraphQLRequestParser<Context> {

-    @Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE")
-    override suspend fun parseRequest(context: Context): GraphQLServerRequest? = try {
-        context.bodyAsClass(GraphQLServerRequest::class.java)
-    } catch (e: IOException) {
-        null
+    @Suppress("PARAMETER_NAME_CHANGED_ON_OVERRIDE", "UNCHECKED_CAST")
+    override suspend fun parseRequest(context: Context): GraphQLServerRequest? {
+        return try {
+            val formParam = context.formParam("operation")
+                ?: return context.bodyAsClass(GraphQLServerRequest::class.java)
+
+            val request = context.jsonMapper().fromJsonString(
+                formParam,
+                GraphQLServerRequest::class.java
+            )
+            val map = context.formParam("map")?.let {
+                context.jsonMapper().fromJsonString(
+                    it,
+                    Map::class.java as Class<Map<String, List<String>>>
+                )
+            }.orEmpty()
+
+            val filesMap = map.keys
+                .sortedBy { it.toIntOrNull() }
+                .map { context.uploadedFile(it) }
+
+            val mapItems = map.flatMap { (index, variables) ->
+                val indexInt = index.toIntOrNull() ?: return@flatMap emptyList()
+                val file = filesMap.getOrNull(indexInt)
+                variables.map { fullVariable ->
+                    val variable = fullVariable.removePrefix("variables.").substringBefore('.')
+                    val listIndex = fullVariable.substringAfterLast('.').toIntOrNull()
+                    MapItem(
+                        indexInt,
+                        variable,
+                        listIndex,
+                        file
+                    )
+                }
+            }.groupBy { it.variable }
+
+            when (request) {
+                is GraphQLRequest -> {
+                    request.copy(variables = request.variables?.modifyFiles(mapItems))
+                }
+                is GraphQLBatchRequest -> {
+                    request.copy(
+                        requests = request.requests.map {
+                            it.copy(
+                                variables = it.variables?.modifyFiles(mapItems)
+                            )
+                        }
+                    )
+                }
+            }
+        } catch (e: IOException) {
+            null
+        }
+    }
+
+    data class MapItem(
+        val index: Int,
+        val variable: String,
+        val listIndex: Int?,
+        val file: UploadedFile?
+    )
+
+    /**
+     * Example [this]: { "file": null }
+     * Example: '{ "query": "mutation ($file: Upload!) { singleUpload(file: $file) { id } }", "variables": { "file": null } }'
+     * Example map "{ "0": ["variables.file"] }"
+     * TODO nested objects
+     */
+    private fun Map<String, Any?>.modifyFiles(map: Map<String, List<MapItem>>): Map<String, Any?> {
+        return mapValues { (name, value) ->
+            if (map.containsKey(name)) {
+                val items = map[name].orEmpty()
+                if (items.size > 1) {
+                    if (value is List<*>) {
+                        value.mapIndexed { index, any ->
+                            any ?: items.firstOrNull { it.listIndex == index }?.file
+                        }
+                    } else {
+                        value
+                    }
+                } else {
+                    value ?: items.firstOrNull()?.file
+                }
+            } else {
+                value
+            }
+        }
    }
 }
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TachideskGraphQLSchema.kt
@@ -12,12 +12,15 @@ import com.expediagroup.graphql.generator.TopLevelObject
 import com.expediagroup.graphql.generator.hooks.FlowSubscriptionSchemaGeneratorHooks
 import com.expediagroup.graphql.generator.toSchema
 import graphql.schema.GraphQLType
+import io.javalin.http.UploadedFile
+import suwayomi.tachidesk.graphql.mutations.BackupMutation
 import suwayomi.tachidesk.graphql.mutations.CategoryMutation
 import suwayomi.tachidesk.graphql.mutations.ChapterMutation
 import suwayomi.tachidesk.graphql.mutations.ExtensionMutation
 import suwayomi.tachidesk.graphql.mutations.MangaMutation
 import suwayomi.tachidesk.graphql.mutations.MetaMutation
 import suwayomi.tachidesk.graphql.mutations.SourceMutation
+import suwayomi.tachidesk.graphql.queries.BackupQuery
 import suwayomi.tachidesk.graphql.queries.CategoryQuery
 import suwayomi.tachidesk.graphql.queries.ChapterQuery
 import suwayomi.tachidesk.graphql.queries.ExtensionQuery
@@ -27,6 +30,7 @@ import suwayomi.tachidesk.graphql.queries.SourceQuery
 import suwayomi.tachidesk.graphql.server.primitives.Cursor
 import suwayomi.tachidesk.graphql.server.primitives.GraphQLCursor
 import suwayomi.tachidesk.graphql.server.primitives.GraphQLLongAsString
+import suwayomi.tachidesk.graphql.server.primitives.GraphQLUpload
 import suwayomi.tachidesk.graphql.subscriptions.DownloadSubscription
 import kotlin.reflect.KClass
 import kotlin.reflect.KType
@@ -35,6 +39,7 @@ class CustomSchemaGeneratorHooks : FlowSubscriptionSchemaGeneratorHooks() {
    override fun willGenerateGraphQLType(type: KType): GraphQLType? = when (type.classifier as? KClass<*>) {
        Long::class -> GraphQLLongAsString // encode to string for JS
        Cursor::class -> GraphQLCursor
+        UploadedFile::class -> GraphQLUpload
        else -> super.willGenerateGraphQLType(type)
    }
 }
@@ -46,6 +51,7 @@ val schema = toSchema(
        hooks = CustomSchemaGeneratorHooks()
    ),
    queries = listOf(
+        TopLevelObject(BackupQuery()),
        TopLevelObject(CategoryQuery()),
        TopLevelObject(ChapterQuery()),
        TopLevelObject(ExtensionQuery()),
@@ -54,6 +60,7 @@ val schema = toSchema(
        TopLevelObject(SourceQuery())
    ),
    mutations = listOf(
+        TopLevelObject(BackupMutation()),
        TopLevelObject(CategoryMutation()),
        TopLevelObject(ChapterMutation()),
        TopLevelObject(ExtensionMutation()),
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TemporaryFileStorage.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/TemporaryFileStorage.kt
@@ -0,0 +1,46 @@
+package suwayomi.tachidesk.graphql.server
+
+import kotlinx.coroutines.DelicateCoroutinesApi
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import java.io.InputStream
+import java.nio.file.Files
+import java.nio.file.Path
+import kotlin.concurrent.thread
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.deleteIfExists
+import kotlin.io.path.deleteRecursively
+import kotlin.io.path.outputStream
+import kotlin.time.Duration.Companion.days
+
+@OptIn(ExperimentalPathApi::class)
+object TemporaryFileStorage {
+    private val folder = Files.createTempDirectory("Tachidesk")
+
+    init {
+        Runtime.getRuntime().addShutdownHook(
+            thread(start = false) {
+                folder.deleteRecursively()
+            }
+        )
+    }
+
+    @OptIn(DelicateCoroutinesApi::class)
+    fun saveFile(name: String, content: InputStream) {
+        val file = folder.resolve(name)
+        content.use { inStream ->
+            file.outputStream().use {
+                inStream.copyTo(it)
+            }
+        }
+        GlobalScope.launch {
+            delay(1.days)
+            file.deleteIfExists()
+        }
+    }
+
+    fun retrieveFile(name: String): Path {
+        return folder.resolve(name)
+    }
+}
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/primitives/Upload.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/server/primitives/Upload.kt
@@ -0,0 +1,60 @@
+package suwayomi.tachidesk.graphql.server.primitives
+
+import graphql.GraphQLContext
+import graphql.scalar.CoercingUtil
+import graphql.schema.Coercing
+import graphql.schema.CoercingParseValueException
+import graphql.schema.CoercingSerializeException
+import graphql.schema.GraphQLScalarType
+import io.javalin.http.UploadedFile
+import java.util.Locale
+
+val GraphQLUpload = GraphQLScalarType.newScalar()
+    .name("Upload")
+    .description("A file part in a multipart request")
+    .coercing(GraphqlUploadCoercing())
+    .build()
+
+private class GraphqlUploadCoercing : Coercing<UploadedFile, Void?> {
+    private fun parseValueImpl(input: Any, locale: Locale): UploadedFile {
+        if (input !is UploadedFile) {
+            throw CoercingParseValueException(
+                CoercingUtil.i18nMsg(
+                    locale,
+                    "String.unexpectedRawValueType",
+                    CoercingUtil.typeName(input)
+                )
+            )
+        }
+        return input
+    }
+
+    @Deprecated("")
+    override fun serialize(dataFetcherResult: Any): Void? {
+        throw CoercingSerializeException("Upload is an input-only type")
+    }
+
+    @Throws(CoercingSerializeException::class)
+    override fun serialize(
+        dataFetcherResult: Any,
+        graphQLContext: GraphQLContext,
+        locale: Locale
+    ): Void? {
+        throw CoercingSerializeException("Upload is an input-only type")
+    }
+
+    @Deprecated("")
+    override fun parseValue(input: Any): UploadedFile {
+        return parseValueImpl(input, Locale.getDefault())
+    }
+
+    @Throws(CoercingParseValueException::class)
+    override fun parseValue(input: Any, graphQLContext: GraphQLContext, locale: Locale): UploadedFile {
+        return parseValueImpl(input, locale)
+    }
+
+    @Deprecated("")
+    override fun parseLiteral(input: Any): UploadedFile {
+        return parseValueImpl(input, Locale.getDefault())
+    }
+}
--- a/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/BackupTypes.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/graphql/types/BackupTypes.kt
@@ -0,0 +1,35 @@
+package suwayomi.tachidesk.graphql.types
+
+import suwayomi.tachidesk.manga.impl.backup.proto.ProtoBackupImport
+
+enum class BackupRestoreState {
+    IDLE,
+    RESTORING_CATEGORIES,
+    RESTORING_MANGA
+}
+
+data class BackupRestoreStatus(
+    val state: BackupRestoreState,
+    val totalManga: Int,
+    val mangaProgress: Int
+)
+
+fun ProtoBackupImport.BackupRestoreState.toStatus(): BackupRestoreStatus {
+    return when (this) {
+        ProtoBackupImport.BackupRestoreState.Idle -> BackupRestoreStatus(
+            state = BackupRestoreState.IDLE,
+            totalManga = 0,
+            mangaProgress = 0
+        )
+        is ProtoBackupImport.BackupRestoreState.RestoringCategories -> BackupRestoreStatus(
+            state = BackupRestoreState.RESTORING_CATEGORIES,
+            totalManga = totalManga,
+            mangaProgress = 0
+        )
+        is ProtoBackupImport.BackupRestoreState.RestoringManga -> BackupRestoreStatus(
+            state = BackupRestoreState.RESTORING_MANGA,
+            totalManga = totalManga,
+            mangaProgress = current
+        )
+    }
+}